Build-time flake inputs

src/libcmd/common-eval-args.cc

@@ -20,10 +20,6 @@ namespace nix {
 
 namespace fs { using namespace std::filesystem; }
 
-fetchers::Settings fetchSettings;
-
-static GlobalConfig::Register rFetchSettings(&fetchSettings);
-
 EvalSettings evalSettings {
     settings.readOnlyMode,
     {
@@ -34,7 +30,12 @@ EvalSettings evalSettings {
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
                 auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
-                auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+                auto storePath = nix::fetchToStore(
+                    state.fetchSettings,
+                    *state.store,
+                    SourcePath(accessor),
+                    FetchMode::Copy,
+                    lockedRef.input.getName());
                 state.allowPath(storePath);
                 return state.storePath(storePath);
             },
@@ -177,14 +178,23 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
             state.store,
             state.fetchSettings,
             EvalSettings::resolvePseudoUrl(s));
-        auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
+        auto storePath = fetchToStore(
+            state.fetchSettings,
+            *state.store,
+            SourcePath(accessor),
+            FetchMode::Copy);
         return state.storePath(storePath);
     }
 
     else if (hasPrefix(s, "flake:")) {
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
         auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
-        auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+        auto storePath = nix::fetchToStore(
+            state.fetchSettings,
+            *state.store,
+            SourcePath(accessor),
+            FetchMode::Copy,
+            lockedRef.input.getName());
         state.allowPath(storePath);
         return state.storePath(storePath);
     }

src/libcmd/include/nix/cmd/common-eval-args.hh

@@ -22,9 +22,6 @@ struct SourcePath;
 
 namespace flake { struct Settings; }
 
-/**
- * @todo Get rid of global setttings variables
- */
 extern fetchers::Settings fetchSettings;
 
 /**

src/libcmd/installable-value.cc

@@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
+        auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libexpr/eval.cc

@@ -2427,6 +2427,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
         ? *dstPathCached
         : [&]() {
             auto dstPath = fetchToStore(
+                fetchSettings,
                 *store,
                 path.resolveSymlinks(SymlinkResolution::Ancestors),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
@@ -3139,7 +3140,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
                 store,
                 fetchSettings,
                 EvalSettings::resolvePseudoUrl(value));
-            auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
+            auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
             return finish(this->storePath(storePath));
         } catch (Error & e) {
             logWarning({

src/libexpr/include/nix/expr/eval.hh

@@ -458,7 +458,8 @@ public:
         fetchers::Input & input,
         const fetchers::Input & originalInput,
         ref<SourceAccessor> accessor,
-        bool requireLockable);
+        bool requireLockable,
+        bool forceNarHash = false);
 
     /**
      * Parse a Nix expression from the specified file.

src/libexpr/paths.cc

@@ -24,7 +24,11 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites)
 {
     if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) {
         auto storePath = fetchToStore(
-            *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name());
+            fetchSettings,
+            *store,
+            SourcePath{ref(mount)},
+            settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
+            path.name());
         assert(storePath.name() == path.name());
         if (rewrites)
             rewrites->emplace(path.hashPart(), storePath.hashPart());
@@ -61,23 +65,29 @@ std::string EvalState::computeBaseName(const SourcePath & path)
                 "This can typically be avoided by rewriting an attribute like `src = ./.` "
                 "to `src = builtins.path { path = ./.; name = \"source\"; }`.",
                 path);
-            return std::string(fetchToStore(*store, path, FetchMode::DryRun, storePath->name()).to_string());
+            return std::string(
+                fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string());
         }
     }
     return std::string(path.baseName());
 }
 
 StorePath EvalState::mountInput(
-    fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor, bool requireLockable)
+    fetchers::Input & input,
+    const fetchers::Input & originalInput,
+    ref<SourceAccessor> accessor,
+    bool requireLockable,
+    bool forceNarHash)
 {
-    auto storePath = settings.lazyTrees ? StorePath::random(input.getName())
-                                        : fetchToStore(*store, accessor, FetchMode::Copy, input.getName());
+    auto storePath = settings.lazyTrees
+                         ? StorePath::random(input.getName())
+                         : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName());
 
     allowPath(storePath); // FIXME: should just whitelist the entire virtual store
 
     storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
 
-    if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) {
+    if ((forceNarHash || (requireLockable && (!settings.lazyTrees || !input.isLocked()))) && !input.getNarHash()) {
         auto narHash = accessor->hashPath(CanonPath::root);
         input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
     }

src/libexpr/primops.cc

@@ -2539,6 +2539,7 @@ static void addPath(
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
             // FIXME: make this lazy?
             auto dstPath = fetchToStore(
+                state.fetchSettings,
                 *state.store,
                 path.resolveSymlinks(),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,

src/libexpr/primops/fetchTree.cc

@@ -533,11 +533,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     auto storePath =
         unpack
         ? fetchToStore(
+            state.fetchSettings,
             *state.store,
             fetchers::downloadTarball(state.store, state.fetchSettings, *url),
             FetchMode::Copy,
             name)
-        : fetchers::downloadFile(state.store, *url, name).storePath;
+        : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
 
     if (expectedHash) {
         auto hash = unpack

src/libfetchers/builtin.cc

@@ -0,0 +1,56 @@
+#include "nix/store/builtins.hh"
+#include "nix/store/parsed-derivations.hh"
+#include "nix/fetchers/fetchers.hh"
+#include "nix/fetchers/fetch-settings.hh"
+#include "nix/util/archive.hh"
+#include "nix/store/filetransfer.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+static void builtinFetchTree(const BuiltinBuilderContext & ctx)
+{
+    auto out = get(ctx.drv.outputs, "out");
+    if (!out)
+        throw Error("'builtin:fetch-tree' requires an 'out' output");
+
+    if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure()))
+        throw Error("'builtin:fetch-tree' must be a fixed-output or impure derivation");
+
+    if (!ctx.structuredAttrs)
+        throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'");
+
+    setenv("NIX_CACHE_HOME", ctx.tmpDirInSandbox.c_str(), 1);
+
+    using namespace fetchers;
+
+    fetchers::Settings myFetchSettings;
+    myFetchSettings.accessTokens = fetchSettings.accessTokens.get();
+
+    // Make sure we don't use the FileTransfer object of the parent
+    // since it's in a broken state after the fork. We also must not
+    // delete it, so hang on to the shared_ptr.
+    // FIXME: move FileTransfer into fetchers::Settings.
+    auto prevFileTransfer = resetFileTransfer();
+
+    // FIXME: disable use of the git/tarball cache
+
+    auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs((*ctx.structuredAttrs)["input"]));
+
+    std::cerr << fmt("fetching '%s'...\n", input.to_string());
+
+    /* Make sure we don't use the real store because we're in a forked
+       process. */
+    auto dummyStore = openStore("dummy://");
+
+    auto [accessor, lockedInput] = input.getAccessor(dummyStore);
+
+    auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); });
+
+    restorePath(ctx.outputs.at("out"), *source);
+}
+
+static RegisterBuiltinBuilder registerUnpackChannel("fetch-tree", builtinFetchTree);
+
+}

src/libfetchers/cache.cc

@@ -1,4 +1,5 @@
 #include "nix/fetchers/cache.hh"
+#include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/users.hh"
 #include "nix/store/sqlite.hh"
 #include "nix/util/sync.hh"
@@ -162,10 +163,12 @@ struct CacheImpl : Cache
     }
 };
 
-ref<Cache> getCache()
+ref<Cache> Settings::getCache() const
 {
-    static auto cache = std::make_shared<CacheImpl>();
-    return ref<Cache>(cache);
+    auto cache(_cache.lock());
+    if (!*cache)
+        *cache = std::make_shared<CacheImpl>();
+    return ref<Cache>(*cache);
 }
 
 }

src/libfetchers/fetch-settings.cc

@@ -1,4 +1,5 @@
 #include "nix/fetchers/fetch-settings.hh"
+#include "nix/util/config-global.hh"
 
 namespace nix::fetchers {
 
@@ -7,3 +8,11 @@ Settings::Settings()
 }
 
 }
+
+namespace nix {
+
+fetchers::Settings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
+}

src/libfetchers/fetch-to-store.cc

@@ -1,13 +1,14 @@
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/fetchers.hh"
+#include "nix/fetchers/fetch-settings.hh"
 
 namespace nix {
 
 fetchers::Cache::Key makeFetchToStoreCacheKey(
-    const std::string &name,
-    const std::string &fingerprint,
+    const std::string & name,
+    const std::string & fingerprint,
     ContentAddressMethod method,
-    const std::string &path)
+    const std::string & path)
 {
     return fetchers::Cache::Key{"fetchToStore", {
         {"name", name},
@@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey(
 }
 
 StorePath fetchToStore(
+    const fetchers::Settings & settings,
     Store & store,
     const SourcePath & path,
     FetchMode mode,
@@ -34,7 +36,7 @@ StorePath fetchToStore(
 
     if (!filter && path.accessor->fingerprint) {
         cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
-        if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) {
+        if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
             debug("store path cache hit for '%s'", path);
             return res->storePath;
         }
@@ -56,7 +58,7 @@ StorePath fetchToStore(
     debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
 
     if (cacheKey && mode == FetchMode::Copy)
-        fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
+        settings.getCache()->upsert(*cacheKey, store, {}, storePath);
 
     return storePath;
 }

src/libfetchers/fetchers.cc

@@ -195,7 +195,7 @@ std::tuple<StorePath, ref<SourceAccessor>, Input> Input::fetchToStore(ref<Store>
     try {
         auto [accessor, result] = getAccessorUnchecked(store);
 
-        auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName());
+        auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
 
         auto narHash = store->queryPathInfo(storePath)->narHash;
         result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));

src/libfetchers/git-utils.cc

@@ -1,6 +1,7 @@
 #include "nix/fetchers/git-utils.hh"
 #include "nix/fetchers/git-lfs-fetch.hh"
 #include "nix/fetchers/cache.hh"
+#include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/finally.hh"
 #include "nix/util/processes.hh"
 #include "nix/util/signals.hh"
@@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
     }
 
-    Hash treeHashToNarHash(const Hash & treeHash) override
+    Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
     {
         auto accessor = getAccessor(treeHash, false, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
-        if (auto res = fetchers::getCache()->lookup(cacheKey))
+        if (auto res = settings.getCache()->lookup(cacheKey))
             return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
 
         auto narHash = accessor->hashPath(CanonPath::root);
 
-        fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
+        settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
 
         return narHash;
     }
@@ -1260,11 +1261,19 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     return result;
 }
 
-ref<GitRepo> getTarballCache()
+namespace fetchers {
+
+ref<GitRepo> Settings::getTarballCache() const
 {
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
+    auto tarballCache(_tarballCache.lock());
+    if (!*tarballCache)
+        *tarballCache = GitRepo::openRepo(
+            std::filesystem::path(getCacheDir()) / "tarball-cache",
+            true,
+            true);
+    return ref<GitRepo>(*tarballCache);
+}
 
-    return GitRepo::openRepo(repoDir, true, true);
 }
 
 GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)