diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 21584b74aff..376dfd6a571 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -20,10 +20,6 @@ namespace nix { namespace fs { using namespace std::filesystem; } -fetchers::Settings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - EvalSettings evalSettings { settings.readOnlyMode, { @@ -34,7 +30,12 @@ EvalSettings evalSettings { auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); - auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + auto storePath = nix::fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy, + lockedRef.input.getName()); state.allowPath(storePath); return state.storePath(storePath); }, @@ -177,14 +178,23 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s)); - auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); + auto storePath = fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy); return state.storePath(storePath); } else if (hasPrefix(s, "flake:")) { auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); - auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + auto storePath = nix::fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy, + lockedRef.input.getName()); state.allowPath(storePath); return state.storePath(storePath); } diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 6f3367e58e9..aefc3cc31e6 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -22,9 +22,6 @@ struct SourcePath; namespace flake { struct Settings; } -/** - * @todo Get rid of global setttings variables - */ extern fetchers::Settings fetchSettings; /** diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 4eb4993b14e..f5a129205c8 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -45,7 +45,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy); + auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 85c044c2fa9..ad7191cb6de 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2427,6 +2427,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat ? *dstPathCached : [&]() { auto dstPath = fetchToStore( + fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, @@ -3139,7 +3140,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat store, fetchSettings, EvalSettings::resolvePseudoUrl(value)); - auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); + auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy); return finish(this->storePath(storePath)); } catch (Error & e) { logWarning({ diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index d82baddb153..9e83a90f72f 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -458,7 +458,8 @@ public: fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, - bool requireLockable); + bool requireLockable, + bool forceNarHash = false); /** * Parse a Nix expression from the specified file. diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 3aaca232829..d782b7586ce 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -24,7 +24,11 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) { if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) { auto storePath = fetchToStore( - *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name()); + fetchSettings, + *store, + SourcePath{ref(mount)}, + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + path.name()); assert(storePath.name() == path.name()); if (rewrites) rewrites->emplace(path.hashPart(), storePath.hashPart()); @@ -61,23 +65,29 @@ std::string EvalState::computeBaseName(const SourcePath & path) "This can typically be avoided by rewriting an attribute like `src = ./.` " "to `src = builtins.path { path = ./.; name = \"source\"; }`.", path); - return std::string(fetchToStore(*store, path, FetchMode::DryRun, storePath->name()).to_string()); + return std::string( + fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string()); } } return std::string(path.baseName()); } StorePath EvalState::mountInput( - fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable) + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable, + bool forceNarHash) { - auto storePath = settings.lazyTrees ? StorePath::random(input.getName()) - : fetchToStore(*store, accessor, FetchMode::Copy, input.getName()); + auto storePath = settings.lazyTrees + ? StorePath::random(input.getName()) + : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); - if (requireLockable && (!settings.lazyTrees || !input.isLocked()) && !input.getNarHash()) { + if ((forceNarHash || (requireLockable && (!settings.lazyTrees || !input.isLocked()))) && !input.getNarHash()) { auto narHash = accessor->hashPath(CanonPath::root); input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c6a97fdaee0..665a3a815ab 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2539,6 +2539,7 @@ static void addPath( if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { // FIXME: make this lazy? auto dstPath = fetchToStore( + state.fetchSettings, *state.store, path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c82fb82c5f7..61704b7e6cd 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -533,11 +533,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto storePath = unpack ? fetchToStore( + state.fetchSettings, *state.store, fetchers::downloadTarball(state.store, state.fetchSettings, *url), FetchMode::Copy, name) - : fetchers::downloadFile(state.store, *url, name).storePath; + : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath; if (expectedHash) { auto hash = unpack diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc new file mode 100644 index 00000000000..9f4e2ac35fa --- /dev/null +++ b/src/libfetchers/builtin.cc @@ -0,0 +1,56 @@ +#include "nix/store/builtins.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/archive.hh" +#include "nix/store/filetransfer.hh" + +#include + +namespace nix { + +static void builtinFetchTree(const BuiltinBuilderContext & ctx) +{ + auto out = get(ctx.drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetch-tree' requires an 'out' output"); + + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) + throw Error("'builtin:fetch-tree' must be a fixed-output or impure derivation"); + + if (!ctx.structuredAttrs) + throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'"); + + setenv("NIX_CACHE_HOME", ctx.tmpDirInSandbox.c_str(), 1); + + using namespace fetchers; + + fetchers::Settings myFetchSettings; + myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); + + // Make sure we don't use the FileTransfer object of the parent + // since it's in a broken state after the fork. We also must not + // delete it, so hang on to the shared_ptr. + // FIXME: move FileTransfer into fetchers::Settings. + auto prevFileTransfer = resetFileTransfer(); + + // FIXME: disable use of the git/tarball cache + + auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs((*ctx.structuredAttrs)["input"])); + + std::cerr << fmt("fetching '%s'...\n", input.to_string()); + + /* Make sure we don't use the real store because we're in a forked + process. */ + auto dummyStore = openStore("dummy://"); + + auto [accessor, lockedInput] = input.getAccessor(dummyStore); + + auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); }); + + restorePath(ctx.outputs.at("out"), *source); +} + +static RegisterBuiltinBuilder registerUnpackChannel("fetch-tree", builtinFetchTree); + +} diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index d369d213f51..9a2531ba526 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/cache.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/util/users.hh" #include "nix/store/sqlite.hh" #include "nix/util/sync.hh" @@ -162,10 +163,12 @@ struct CacheImpl : Cache } }; -ref getCache() +ref Settings::getCache() const { - static auto cache = std::make_shared(); - return ref(cache); + auto cache(_cache.lock()); + if (!*cache) + *cache = std::make_shared(); + return ref(*cache); } } diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index 4b4e4e29d98..27526a79705 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/config-global.hh" namespace nix::fetchers { @@ -7,3 +8,11 @@ Settings::Settings() } } + +namespace nix { + +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +} diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index f1b02f4e0a8..f7ab32322ef 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -1,13 +1,14 @@ #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix { fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string &name, - const std::string &fingerprint, + const std::string & name, + const std::string & fingerprint, ContentAddressMethod method, - const std::string &path) + const std::string & path) { return fetchers::Cache::Key{"fetchToStore", { {"name", name}, @@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey( } StorePath fetchToStore( + const fetchers::Settings & settings, Store & store, const SourcePath & path, FetchMode mode, @@ -34,7 +36,7 @@ StorePath fetchToStore( if (!filter && path.accessor->fingerprint) { cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs()); - if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) { + if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) { debug("store path cache hit for '%s'", path); return res->storePath; } @@ -56,7 +58,7 @@ StorePath fetchToStore( debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); if (cacheKey && mode == FetchMode::Copy) - fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); + settings.getCache()->upsert(*cacheKey, store, {}, storePath); return storePath; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 33301933ca5..91c809f8e70 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -195,7 +195,7 @@ std::tuple, Input> Input::fetchToStore(ref try { auto [accessor, result] = getAccessorUnchecked(store); - auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName()); + auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName()); auto narHash = store->queryPathInfo(storePath)->narHash; result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 7e1f085f599..4553511bac4 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/git-utils.hh" #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/cache.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/util/finally.hh" #include "nix/util/processes.hh" #include "nix/util/signals.hh" @@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); } - Hash treeHashToNarHash(const Hash & treeHash) override + Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override { auto accessor = getAccessor(treeHash, false, ""); fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; - if (auto res = fetchers::getCache()->lookup(cacheKey)) + if (auto res = settings.getCache()->lookup(cacheKey)) return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); auto narHash = accessor->hashPath(CanonPath::root); - fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); + settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); return narHash; } @@ -1260,11 +1261,19 @@ std::vector> GitRepoImpl::getSubmodules return result; } -ref getTarballCache() +namespace fetchers { + +ref Settings::getTarballCache() const { - static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; + auto tarballCache(_tarballCache.lock()); + if (!*tarballCache) + *tarballCache = GitRepo::openRepo( + std::filesystem::path(getCacheDir()) / "tarball-cache", + true, + true); + return ref(*tarballCache); +} - return GitRepo::openRepo(repoDir, true, true); } GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ef74397ff90..a38d8fbe21b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -483,11 +483,11 @@ struct GitInputScheme : InputScheme return repoInfo; } - uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const + uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}}; - auto cache = getCache(); + auto cache = settings.getCache(); if (auto res = cache->lookup(key)) return getIntAttr(*res, "lastModified"); @@ -499,11 +499,11 @@ struct GitInputScheme : InputScheme return lastModified; } - uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const + uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; - auto cache = getCache(); + auto cache = settings.getCache(); if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -679,12 +679,12 @@ struct GitInputScheme : InputScheme Attrs infoAttrs({ {"rev", rev.gitRev()}, - {"lastModified", getLastModified(repoInfo, repoDir, rev)}, + {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)}, }); if (!getShallowAttr(input)) infoAttrs.insert_or_assign("revCount", - getRevCount(repoInfo, repoDir, rev)); + getRevCount(*input.settings, repoInfo, repoDir, rev)); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -800,7 +800,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("revCount", - rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev)); + rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev)); verifyCommit(input, repo); } else { @@ -819,7 +819,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign( "lastModified", repoInfo.workdirInfo.headRev - ? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev) + ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev) : 0); return {accessor, std::move(input)}; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index bb82f751fd7..77549b84c21 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -262,7 +262,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.erase("ref"); input.attrs.insert_or_assign("rev", rev->gitRev()); - auto cache = getCache(); + auto cache = input.settings->getCache(); Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; @@ -271,7 +271,7 @@ struct GitArchiveInputScheme : InputScheme if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); - if (getTarballCache()->hasObject(treeHash)) + if (input.settings->getTarballCache()->hasObject(treeHash)) return {std::move(input), TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; else debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); @@ -291,7 +291,7 @@ struct GitArchiveInputScheme : InputScheme fmt("unpacking '%s' into the Git cache", input.to_string())); TarArchive archive { *source }; - auto tarballCache = getTarballCache(); + auto tarballCache = input.settings->getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -327,7 +327,7 @@ struct GitArchiveInputScheme : InputScheme #endif input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); - auto accessor = getTarballCache()->getAccessor( + auto accessor = input.settings->getTarballCache()->getAccessor( tarballInfo.treeHash, false, "«" + input.to_string() + "»"); @@ -406,7 +406,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", headers).storePath))); + downloadFile(store, *input.settings, url, "source", headers).storePath))); return RefInfo { .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), @@ -480,7 +480,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", headers).storePath))); + downloadFile(store, *input.settings, url, "source", headers).storePath))); if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { return RefInfo { @@ -550,7 +550,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath); + downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; getline(is, line); @@ -566,7 +566,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::regex refRegex(refUri); auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath); + downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 5b9319d774b..6ac693183f9 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -91,6 +91,4 @@ struct Cache Store & store) = 0; }; -ref getCache(); - } diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 831a18bf0cd..e1c7f70cff4 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -3,14 +3,24 @@ #include "nix/util/types.hh" #include "nix/util/configuration.hh" +#include "nix/util/ref.hh" +#include "nix/util/sync.hh" #include #include #include +namespace nix { + +struct GitRepo; + +} + namespace nix::fetchers { +struct Cache; + struct Settings : public Config { Settings(); @@ -106,6 +116,24 @@ struct Settings : public Config When empty, disables the global flake registry. )"}; + + ref getCache() const; + + ref getTarballCache() const; + +private: + mutable Sync> _cache; + + mutable Sync> _tarballCache; }; } + +namespace nix { + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +} diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh index 44c33c147ed..a52d567ecfb 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh @@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy }; * Copy the `path` to the Nix store. */ StorePath fetchToStore( + const fetchers::Settings & settings, Store & store, const SourcePath & path, FetchMode mode, diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 1506f8509e4..9cab72b7314 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -5,7 +5,7 @@ namespace nix { -namespace fetchers { struct PublicKey; } +namespace fetchers { struct PublicKey; struct Settings; } /** * A sink that writes into a Git repository. Note that nothing may be written @@ -115,7 +115,7 @@ struct GitRepo * Given a Git tree hash, compute the hash of its NAR * serialisation. This is memoised on-disk. */ - virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0; /** * If the specified Git object is a directory with a single entry @@ -125,8 +125,6 @@ struct GitRepo virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; -ref getTarballCache(); - // A helper to ensure that the `git_*_free` functions get called. template struct Deleter diff --git a/src/libfetchers/include/nix/fetchers/tarball.hh b/src/libfetchers/include/nix/fetchers/tarball.hh index 691142091fa..2c5ea209f01 100644 --- a/src/libfetchers/include/nix/fetchers/tarball.hh +++ b/src/libfetchers/include/nix/fetchers/tarball.hh @@ -26,6 +26,7 @@ struct DownloadFileResult DownloadFileResult downloadFile( ref store, + const Settings & settings, const std::string & url, const std::string & name, const Headers & headers = {}); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index eb6bdd1ebdc..a35e7f47646 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme }}; if (!input.getRev()) { - if (auto res = getCache()->lookupWithTTL(refToRevKey)) + if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey)) input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); } /* If we have a rev, check if we have a cached store path. */ if (auto rev = input.getRev()) { - if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store)) + if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store)) return makeResult(res->value, res->storePath); } @@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme /* Now that we have the rev, check the cache again for a cached store path. */ - if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store)) + if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store)) return makeResult(res->value, res->storePath); Path tmpDir = createTempDir(); @@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme }); if (!origRev) - getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); + input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); - getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); + input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); return makeResult(infoAttrs, std::move(storePath)); } diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 321146ca4ed..cacb2e4a03f 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -34,6 +34,7 @@ subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', + 'builtin.cc', 'cache.cc', 'fetch-settings.cc', 'fetch-to-store.cc', diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ff39cb02f9d..38b1918280c 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -4,6 +4,7 @@ #include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/cache.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix::fetchers { @@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme auto fp = getFingerprint(store, input); if (fp) { auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); - fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath); + input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath); } /* Trust the lastModified value supplied by the user, if diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index bfaf9569a4e..335935f53af 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -156,7 +156,7 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, re } if (!isAbsolute(path)) { - auto storePath = downloadFile(store, path, "flake-registry.json").storePath; + auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ef91d6b2553..96b5318821d 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -9,11 +9,13 @@ #include "nix/fetchers/store-path-accessor.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/git-utils.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix::fetchers { DownloadFileResult downloadFile( ref store, + const Settings & settings, const std::string & url, const std::string & name, const Headers & headers) @@ -25,7 +27,7 @@ DownloadFileResult downloadFile( {"name", name}, }}}; - auto cached = getCache()->lookupStorePath(key, *store); + auto cached = settings.getCache()->lookupStorePath(key, *store); auto useCached = [&]() -> DownloadFileResult { @@ -92,7 +94,7 @@ DownloadFileResult downloadFile( key.second.insert_or_assign("url", url); assert(!res.urls.empty()); infoAttrs.insert_or_assign("url", *res.urls.rbegin()); - getCache()->upsert(key, *store, infoAttrs, *storePath); + settings.getCache()->upsert(key, *store, infoAttrs, *storePath); } return { @@ -104,13 +106,14 @@ DownloadFileResult downloadFile( } static DownloadTarballResult downloadTarball_( + const Settings & settings, const std::string & url, const Headers & headers, const std::string & displayPrefix) { Cache::Key cacheKey{"tarball", {{"url", url}}}; - auto cached = getCache()->lookupExpired(cacheKey); + auto cached = settings.getCache()->lookupExpired(cacheKey); auto attrsToResult = [&](const Attrs & infoAttrs) { @@ -119,11 +122,11 @@ static DownloadTarballResult downloadTarball_( .treeHash = treeHash, .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), - .accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix), + .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix), }; }; - if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) + if (cached && !settings.getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) cached.reset(); if (cached && !cached->expired) @@ -169,7 +172,7 @@ static DownloadTarballResult downloadTarball_( TarArchive{path}; }) : TarArchive{*source}; - auto tarballCache = getTarballCache(); + auto tarballCache = settings.getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -196,7 +199,7 @@ static DownloadTarballResult downloadTarball_( /* Insert a cache entry for every URL in the redirect chain. */ for (auto & url : res->urls) { cacheKey.second.insert_or_assign("url", url); - getCache()->upsert(cacheKey, infoAttrs); + settings.getCache()->upsert(cacheKey, infoAttrs); } // FIXME: add a cache entry for immutableUrl? That could allow @@ -341,7 +344,7 @@ struct FileInputScheme : CurlInputScheme the Nix store directly, since there is little deduplication benefit in using the Git cache for single big files like tarballs. */ - auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName()); + auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName()); auto narHash = store->queryPathInfo(file.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); @@ -373,6 +376,7 @@ struct TarballInputScheme : CurlInputScheme auto input(_input); auto result = downloadTarball_( + *input.settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); @@ -390,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); input.attrs.insert_or_assign("narHash", - getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true)); + input.settings->getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true)); return {result.accessor, input}; } diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index fe326291f1f..4dbf454801c 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -50,7 +50,17 @@ let if node ? parent then parentDir + ("/" + dir) else dir; sourceInfo = - if overrides ? ${key} then + if node.buildTime or false then + derivation { + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = node.locked; + outputHashMode = "recursive"; + outputHash = node.locked.narHash; + } + else if overrides ? ${key} then overrides.${key}.sourceInfo else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo @@ -97,6 +107,7 @@ let result = if node.flake or true then assert builtins.isFunction flake.outputs; + assert !(node.buildTime or false); result else sourceInfo; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index a85acf4b282..59d1de58669 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -99,6 +99,7 @@ static FlakeInput parseFlakeInput( auto sUrl = state.symbols.create("url"); auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); + auto sBuildTime = state.symbols.create("buildTime"); fetchers::Attrs attrs; std::optional url; @@ -123,6 +124,9 @@ static FlakeInput parseFlakeInput( } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); + } else if (attr.name == sBuildTime) { + expectType(state, nBool, *attr.value, attr.pos); + input.buildTime = attr.value->boolean(); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; } else if (attr.name == sFollows) { @@ -260,7 +264,7 @@ static Flake readFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy); + auto storePath = fetchToStore(state.fetchSettings, *state.store, setting.value->path(), FetchMode::Copy); flake.config.settings.emplace( state.symbols[setting.name], state.store->printStorePath(storePath)); @@ -593,6 +597,7 @@ LockedFlake lockFlake( oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake, + oldLock->buildTime, oldLock->parentInputAttrPath); node->inputs.insert_or_assign(id, childNode); @@ -702,6 +707,7 @@ LockedFlake lockFlake( inputFlake.lockedRef, ref, true, + input.buildTime, overridenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -745,13 +751,13 @@ LockedFlake lockFlake( warnRegistry(resolvedRef); return { - state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true)), + state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), lockedRef }; } }(); - auto childNode = make_ref(lockedRef, ref, false, overridenParentPath); + auto childNode = make_ref(lockedRef, ref, false, input.buildTime, overridenParentPath); nodePaths.emplace(childNode, path); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index fdac4397f65..577176f0b9f 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -43,12 +43,18 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; + /** - * true = process flake to get outputs - * - * false = (fetched) static source path + * Whether to call the `flake.nix` file in this input to get its outputs. */ bool isFlake = true; + + /** + * Whether to fetch this input at evaluation time or at build + * time. + */ + bool buildTime = false; + std::optional follows; FlakeInputs overrides; }; diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh index 97bd7a49538..a1a95780706 100644 --- a/src/libflake/include/nix/flake/lockfile.hh +++ b/src/libflake/include/nix/flake/lockfile.hh @@ -37,6 +37,7 @@ struct LockedNode : Node { FlakeRef lockedRef, originalRef; bool isFlake = true; + bool buildTime = false; /* The node relative to which relative source paths (e.g. 'path:../foo') are interpreted. */ @@ -46,10 +47,12 @@ struct LockedNode : Node const FlakeRef & lockedRef, const FlakeRef & originalRef, bool isFlake = true, + bool buildTime = false, std::optional parentInputAttrPath = {}) : lockedRef(std::move(lockedRef)) , originalRef(std::move(originalRef)) , isFlake(isFlake) + , buildTime(buildTime) , parentInputAttrPath(std::move(parentInputAttrPath)) { } diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 646516caf2a..690c1a49468 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -44,6 +44,7 @@ LockedNode::LockedNode( : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) , parentInputAttrPath(json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) { @@ -216,6 +217,8 @@ std::pair LockFile::toJSON() const n["locked"].erase("__final"); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; if (lockedNode->parentInputAttrPath) n["parent"] = *lockedNode->parentInputAttrPath; } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c3b80bb0b9b..bd079f5cb75 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "nix/store/builtins/buildenv.hh" +#include "nix/store/builtins.hh" #include "nix/store/derivations.hh" #include "nix/util/signals.hh" @@ -166,17 +167,15 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinBuildenv(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - auto out = outputs.at("out"); + auto out = ctx.outputs.at("out"); createDirs(out); /* Convert the stuff we get from the environment back into a @@ -203,4 +202,6 @@ void builtinBuildenv( createSymlink(getAttr("manifest"), out + "/manifest.nix"); } +static RegisterBuiltinBuilder registerBuildenv("buildenv", builtinBuildenv); + } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 82f268d807d..18fa755580f 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -6,33 +6,29 @@ namespace nix { -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData) +static void builtinFetchurl(const BuiltinBuilderContext & ctx) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ - if (netrcData != "") { + if (ctx.netrcData != "") { settings.netrcFile = "netrc"; - writeFile(settings.netrcFile, netrcData, 0600); + writeFile(settings.netrcFile, ctx.netrcData, 0600); } settings.caFile = "ca-certificates.crt"; - writeFile(settings.caFile, caFileData, 0600); + writeFile(settings.caFile, ctx.caFileData, 0600); - auto out = get(drv.outputs, "out"); + auto out = get(ctx.drv.outputs, "out"); if (!out) throw Error("'builtin:fetchurl' requires an 'out' output"); - if (!(drv.type().isFixed() || drv.type().isImpure())) + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation"); - auto storePath = outputs.at("out"); - auto mainUrl = drv.env.at("url"); - bool unpack = getOr(drv.env, "unpack", "") == "1"; + auto storePath = ctx.outputs.at("out"); + auto mainUrl = ctx.drv.env.at("url"); + bool unpack = getOr(ctx.drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ @@ -56,8 +52,8 @@ void builtinFetchurl( else writeFile(storePath, *source); - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { + auto executable = ctx.drv.env.find("executable"); + if (executable != ctx.drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) throw SysError("making '%1%' executable", storePath); } @@ -79,4 +75,6 @@ void builtinFetchurl( fetch(mainUrl); } +static RegisterBuiltinBuilder registerFetchurl("fetchurl", builtinFetchurl); + } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index f6be21e356b..6247d0a505d 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -5,17 +5,15 @@ namespace nix { namespace fs { using namespace std::filesystem; } -void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs) +static void builtinUnpackChannel(const BuiltinBuilderContext & ctx) { auto getAttr = [&](const std::string & name) -> const std::string & { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + auto i = ctx.drv.env.find(name); + if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; - fs::path out{outputs.at("out")}; + fs::path out{ctx.outputs.at("out")}; auto & channelName = getAttr("channelName"); auto & src = getAttr("src"); @@ -48,4 +46,6 @@ void builtinUnpackChannel( } } +static RegisterBuiltinBuilder registerUnpackChannel("unpack-channel", builtinUnpackChannel); + } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index fb7c6c7a24a..b1d334e1a2f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -798,24 +798,29 @@ struct curlFileTransfer : public FileTransfer } }; -ref makeCurlFileTransfer() -{ - return make_ref(); -} +static Sync> _fileTransfer; ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer.lock()); - if (fileTransfer->state_.lock()->quit) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->quit) + *fileTransfer = std::make_shared(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer() { - return makeCurlFileTransfer(); + return make_ref(); +} + +std::shared_ptr resetFileTransfer() +{ + auto fileTransfer(_fileTransfer.lock()); + std::shared_ptr prev; + fileTransfer->swap(prev); + return prev; } std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index 004e9ef64a2..974c582c92b 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -3,17 +3,32 @@ #include "nix/store/derivations.hh" +#include + namespace nix { -// TODO: make pluggable. -void builtinFetchurl( - const BasicDerivation & drv, - const std::map & outputs, - const std::string & netrcData, - const std::string & caFileData); +struct BuiltinBuilderContext +{ + BasicDerivation & drv; + nlohmann::json * structuredAttrs; + std::map outputs; + std::string netrcData; + std::string caFileData; + Path tmpDirInSandbox; +}; + +using BuiltinBuilder = std::function; + +struct RegisterBuiltinBuilder +{ + typedef std::map BuiltinBuilders; + static BuiltinBuilders * builtinBuilders; -void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs); + RegisterBuiltinBuilder(const std::string & name, BuiltinBuilder && fun) + { + if (!builtinBuilders) builtinBuilders = new BuiltinBuilders; + builtinBuilders->insert_or_assign(name, std::move(fun)); + } +}; } diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index a0a26203716..163666c0bd4 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -45,8 +45,4 @@ typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -void builtinBuildenv( - const BasicDerivation & drv, - const std::map & outputs); - } diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index f87f68e7fc8..9d4f8e7eb29 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -169,6 +169,8 @@ ref getFileTransfer(); */ ref makeFileTransfer(); +std::shared_ptr resetFileTransfer(); + class FileTransferError : public Error { public: diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index d65db6133ba..cfa6d852bff 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -44,6 +44,11 @@ public: } std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); + + nlohmann::json * getStructuredAttrs() + { + return structuredAttrs.get(); + } }; std::string writeStructuredAttrsShell(const nlohmann::json & json); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 3ba1e823f06..e79410c6ac7 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -80,6 +80,8 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, namespace nix { +RegisterBuiltinBuilder::BuiltinBuilders * RegisterBuiltinBuilder::builtinBuilders = nullptr; + void handleDiffHook( uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, @@ -1804,15 +1806,19 @@ void LocalDerivationGoal::runChild() /* Make the contents of netrc and the CA certificate bundle available to builtin:fetchurl (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - std::string caFileData; + BuiltinBuilderContext ctx{ + .drv = *drv, + .structuredAttrs = parsedDrv->getStructuredAttrs(), + .tmpDirInSandbox = tmpDirInSandbox, + }; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { try { - netrcData = readFile(settings.netrcFile); + ctx.netrcData = readFile(settings.netrcFile); } catch (SystemError &) { } try { - caFileData = readFile(settings.caFile); + ctx.caFileData = readFile(settings.caFile); } catch (SystemError &) { } } @@ -2232,19 +2238,16 @@ void LocalDerivationGoal::runChild() try { logger = makeJSONLogger(getStandardError()); - std::map outputs; for (auto & e : drv->outputs) - outputs.insert_or_assign(e.first, + ctx.outputs.insert_or_assign(e.first, worker.store.printStorePath(scratchOutputs.at(e.first))); - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, outputs, netrcData, caFileData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(*drv, outputs); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(*drv, outputs); + std::string builtinName = drv->builder.substr(8); + assert(RegisterBuiltinBuilder::builtinBuilders); + if (auto builtin = get(*RegisterBuiltinBuilder::builtinBuilders, builtinName)) + (*builtin)(ctx); else - throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); + throw Error("unsupported builtin builder '%1%'", builtinName); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 0c3e1f52836..4b9d546d2b7 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -39,6 +39,7 @@ public: SyncBase() { } SyncBase(const T & data) : data(data) { } SyncBase(T && data) noexcept : data(std::move(data)) { } + SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { } template class Lock diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 2ad88dbbea3..9ad87a76e84 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -7,6 +7,7 @@ #include "nix/expr/eval-settings.hh" // for defexpr #include "nix/util/users.hh" #include "nix/fetchers/tarball.hh" +#include "nix/fetchers/fetch-settings.hh" #include "self-exe.hh" #include "man-pages.hh" @@ -114,7 +115,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url))); + auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url))); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -128,9 +129,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); + filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); + filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); } } // Regardless of where it came from, add the expression representing this channel to accumulated expression diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4782cbb290f..fba4419651f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1477,7 +1477,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); auto [accessor, lockedRef] = resolvedRef.lazyFetch(store); - auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName()); + auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName()); auto hash = store->queryPathInfo(storePath)->narHash; if (json) { diff --git a/tests/functional/flakes/build-time-flake-inputs.sh b/tests/functional/flakes/build-time-flake-inputs.sh new file mode 100644 index 00000000000..fd28c1d7818 --- /dev/null +++ b/tests/functional/flakes/build-time-flake-inputs.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit + +lazy="$TEST_ROOT/lazy" +createGitRepo "$lazy" +echo world > "$lazy/who" +git -C "$lazy" add who +git -C "$lazy" commit -a -m foo + +repo="$TEST_ROOT/repo" + +createGitRepo "$repo" + +cat > "$repo/flake.nix" < "$lazy/who" +git -C "$lazy" commit -a -m foo + +nix flake update --flake "$repo" + +clearStore + +nix build --out-link "$TEST_ROOT/result" -L "$repo" +[[ $(cat "$TEST_ROOT/result") = utrecht ]] + +rm -rf "$lazy" + +clearStore + +expectStderr 1 nix build --out-link "$TEST_ROOT/result" -L "$repo" | grepQuiet "Cannot build.*source.drv" diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index 213c388a6d9..531d2ce79f7 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -33,6 +33,7 @@ suites += { 'debugger.sh', 'source-paths.sh', 'old-lockfiles.sh', + 'build-time-flake-inputs.sh', ], 'workdir': meson.current_source_dir(), }