Merge pull request #49 from DeterminateSystems/build-time-fetch-tree

Build-time flake inputs

Author: edolstra

src/libcmd/common-eval-args.cc

@@ -18,10 +18,6 @@
 
 namespace nix {
 
-fetchers::Settings fetchSettings;
-
-static GlobalConfig::Register rFetchSettings(&fetchSettings);
-
 EvalSettings evalSettings{
     settings.readOnlyMode,
     {

src/libcmd/include/nix/cmd/common-eval-args.hh

@@ -25,9 +25,6 @@ namespace flake {
 struct Settings;
 }
 
-/**
- * @todo Get rid of global settings variables
- */
 extern fetchers::Settings fetchSettings;
 
 /**

src/libexpr/include/nix/expr/eval.hh

@@ -484,7 +484,8 @@ public:
         fetchers::Input & input,
         const fetchers::Input & originalInput,
         ref<SourceAccessor> accessor,
-        bool requireLockable);
+        bool requireLockable,
+        bool forceNarHash = false);
 
     /**
      * Parse a Nix expression from the specified file.

src/libexpr/paths.cc

@@ -73,7 +73,11 @@ std::string EvalState::computeBaseName(const SourcePath & path, PosIdx pos)
 }
 
 StorePath EvalState::mountInput(
-    fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor, bool requireLockable)
+    fetchers::Input & input,
+    const fetchers::Input & originalInput,
+    ref<SourceAccessor> accessor,
+    bool requireLockable,
+    bool forceNarHash)
 {
     auto storePath = settings.lazyTrees
                          ? StorePath::random(input.getName())
@@ -95,7 +99,9 @@ StorePath EvalState::mountInput(
 
     storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
 
-    if (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked()) && !input.getNarHash())
+    if (forceNarHash
+        || (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked())
+            && !input.getNarHash()))
         input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true));
 
     if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash())

src/libfetchers/builtin.cc

@@ -0,0 +1,60 @@
+#include "nix/store/builtins.hh"
+#include "nix/store/parsed-derivations.hh"
+#include "nix/fetchers/fetchers.hh"
+#include "nix/fetchers/fetch-settings.hh"
+#include "nix/util/archive.hh"
+#include "nix/store/filetransfer.hh"
+#include "nix/store/store-open.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+static void builtinFetchTree(const BuiltinBuilderContext & ctx)
+{
+    experimentalFeatureSettings.require(Xp::BuildTimeFetchTree);
+
+    auto out = get(ctx.drv.outputs, "out");
+    if (!out)
+        throw Error("'builtin:fetch-tree' requires an 'out' output");
+
+    if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure()))
+        throw Error("'builtin:fetch-tree' must be a fixed-output or impure derivation");
+
+    if (!ctx.parsedDrv)
+        throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'");
+
+    setenv("NIX_CACHE_HOME", ctx.tmpDirInSandbox.c_str(), 1);
+
+    using namespace fetchers;
+
+    fetchers::Settings myFetchSettings;
+    myFetchSettings.accessTokens = fetchSettings.accessTokens.get();
+
+    // Make sure we don't use the FileTransfer object of the parent
+    // since it's in a broken state after the fork. We also must not
+    // delete it, so hang on to the shared_ptr.
+    // FIXME: move FileTransfer into fetchers::Settings.
+    static auto prevFileTransfer = resetFileTransfer();
+
+    // FIXME: disable use of the git/tarball cache
+
+    auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs(ctx.parsedDrv->structuredAttrs["input"]));
+
+    std::cerr << fmt("fetching '%s'...\n", input.to_string());
+
+    /* Functions like downloadFile() expect a store. We can't use the
+       real one since we're in a forked process. FIXME: use recursive
+       Nix's daemon so we can use the real store? */
+    auto tmpStore = openStore(ctx.tmpDirInSandbox + "/nix");
+
+    auto [accessor, lockedInput] = input.getAccessor(tmpStore);
+
+    auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); });
+
+    restorePath(ctx.outputs.at("out"), *source);
+}
+
+static RegisterBuiltinBuilder registerUnpackChannel("fetch-tree", builtinFetchTree);
+
+} // namespace nix

src/libfetchers/fetch-settings.cc

@@ -1,7 +1,16 @@
 #include "nix/fetchers/fetch-settings.hh"
+#include "nix/util/config-global.hh"
 
 namespace nix::fetchers {
 
 Settings::Settings() {}
 
 } // namespace nix::fetchers
+
+namespace nix {
+
+fetchers::Settings fetchSettings;
+
+static GlobalConfig::Register rFetchSettings(&fetchSettings);
+
+} // namespace nix

src/libfetchers/git-utils.cc

@@ -10,6 +10,7 @@
 #include "nix/util/sync.hh"
 #include "nix/util/thread-pool.hh"
 #include "nix/util/pool.hh"
+#include "nix/util/executable-path.hh"
 
 #include <git2/attr.h>
 #include <git2/blob.h>
@@ -549,21 +550,44 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         // that)
         //       then use code that was removed in this commit (see blame)
 
-        auto dir = this->path;
-        Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"};
-        if (shallow)
-            append(gitArgs, {"--depth", "1"});
-        append(gitArgs, {std::string("--"), url, refspec});
-
-        runProgram(
-            RunOptions{
-                .program = "git",
-                .lookupPath = true,
-                // FIXME: git stderr messes up our progress indicator, so
-                // we're using --quiet for now. Should process its stderr.
-                .args = gitArgs,
-                .input = {},
-                .isInteractive = true});
+        if (ExecutablePath::load().findName("git")) {
+            auto dir = this->path;
+            Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"};
+            if (shallow)
+                append(gitArgs, {"--depth", "1"});
+            append(gitArgs, {std::string("--"), url, refspec});
+
+            runProgram(
+                RunOptions{
+                    .program = "git",
+                    .lookupPath = true,
+                    // FIXME: git stderr messes up our progress indicator, so
+                    // we're using --quiet for now. Should process its stderr.
+                    .args = gitArgs,
+                    .input = {},
+                    .isInteractive = true});
+        } else {
+            // Fall back to using libgit2 for fetching. This does not
+            // support SSH very well.
+            Remote remote;
+
+            if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
+                throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message);
+
+            char * refspecs[] = {(char *) refspec.c_str()};
+            git_strarray refspecs2{.strings = refspecs, .count = 1};
+
+            git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
+            // FIXME: for some reason, shallow fetching over ssh barfs
+            // with "could not read from remote repository".
+            opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL;
+            opts.callbacks.payload = &act;
+            opts.callbacks.sideband_progress = sidebandProgressCallback;
+            opts.callbacks.transfer_progress = transferProgressCallback;
+
+            if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr))
+                throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
+        }
     }
 
     void verifyCommit(const Hash & rev, const std::vector<fetchers::PublicKey> & publicKeys) override
@@ -1312,13 +1336,18 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     return result;
 }
 
-ref<GitRepo> getTarballCache()
-{
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
+namespace fetchers {
 
-    return GitRepo::openRepo(repoDir, true, true);
+ref<GitRepo> Settings::getTarballCache() const
+{
+    auto tarballCache(_tarballCache.lock());
+    if (!*tarballCache)
+        *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
+    return ref<GitRepo>(*tarballCache);
 }
 
+} // namespace fetchers
+
 GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)
 {
     static Sync<std::map<std::filesystem::path, WorkdirInfo>> _cache;

src/libfetchers/github.cc

@@ -269,7 +269,7 @@ struct GitArchiveInputScheme : InputScheme
             if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
                 auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
                 auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
-                if (getTarballCache()->hasObject(treeHash))
+                if (input.settings->getTarballCache()->hasObject(treeHash))
                     return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}};
                 else
                     debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev());
@@ -289,7 +289,7 @@ struct GitArchiveInputScheme : InputScheme
             *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string()));
 
         TarArchive archive{*source};
-        auto tarballCache = getTarballCache();
+        auto tarballCache = input.settings->getTarballCache();
         auto parseSink = tarballCache->getFileSystemObjectSink();
         auto lastModified = unpackTarfileToSink(archive, *parseSink);
         auto tree = parseSink->flush();
@@ -323,7 +323,8 @@ struct GitArchiveInputScheme : InputScheme
 #endif
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
-        auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
+        auto accessor =
+            input.settings->getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
 
         if (!input.settings->trustTarballsFromGitForges)
             // FIXME: computing the NAR hash here is wasteful if

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -11,6 +11,12 @@
 
 #include <sys/types.h>
 
+namespace nix {
+
+struct GitRepo;
+
+}
+
 namespace nix::fetchers {
 
 struct Cache;
@@ -119,8 +125,21 @@ struct Settings : public Config
 
     ref<Cache> getCache() const;
 
+    ref<GitRepo> getTarballCache() const;
+
 private:
     mutable Sync<std::shared_ptr<Cache>> _cache;
+
+    mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
 };
 
 } // namespace nix::fetchers
+
+namespace nix {
+
+/**
+ * @todo Get rid of global setttings variables
+ */
+extern fetchers::Settings fetchSettings;
+
+} // namespace nix

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -120,8 +120,6 @@ struct GitRepo
     virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0;
 };
 
-ref<GitRepo> getTarballCache();
-
 // A helper to ensure that the `git_*_free` functions get called.
 template<auto del>
 struct Deleter