Merge pull request #27 from DeterminateSystems/lazy-trees-v2

Lazy trees v2

Author: edolstra

.github/workflows/ci.yml

@@ -91,7 +91,30 @@ jobs:
   flake_regressions:
     if: github.event_name == 'merge_group'
     needs: build_x86_64-linux
-    runs-on: blacksmith-32vcpu-ubuntu-2204
+    runs-on: namespace-profile-x86-32cpu-64gb
+    steps:
+      - name: Checkout nix
+        uses: actions/checkout@v4
+      - name: Checkout flake-regressions
+        uses: actions/checkout@v4
+        with:
+          repository: DeterminateSystems/flake-regressions
+          path: flake-regressions
+      - name: Checkout flake-regressions-data
+        uses: actions/checkout@v4
+        with:
+          repository: DeterminateSystems/flake-regressions-data
+          path: flake-regressions/tests
+      - uses: DeterminateSystems/nix-installer-action@main
+        with:
+          determinate: true
+      - uses: DeterminateSystems/flakehub-cache-action@main
+      - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh
+
+  flake_regressions_lazy:
+    if: github.event_name == 'merge_group'
+    needs: build_x86_64-linux
+    runs-on: namespace-profile-x86-32cpu-64gb
     steps:
       - name: Checkout nix
         uses: actions/checkout@v4
@@ -109,7 +132,7 @@ jobs:
         with:
           determinate: true
       - uses: DeterminateSystems/flakehub-cache-action@main
-      - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=50 flake-regressions/eval-all.sh
+      - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh
 
   manual:
     if: github.event_name != 'merge_group'

src/libcmd/installable-value.cc

@@ -57,7 +57,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
     else if (v.type() == nString) {
         return {{
             .path = DerivedPath::fromSingle(
-                state->coerceToSingleDerivedPath(pos, v, errorCtx)),
+                state->devirtualize(
+                    state->coerceToSingleDerivedPath(pos, v, errorCtx))),
             .info = make_ref<ExtraPathInfo>(),
         }};
     }

src/libexpr/eval.cc

@@ -20,6 +20,7 @@
 #include "nix/util/url.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/tarball.hh"
+#include "nix/fetchers/input-cache.hh"
 
 #include "parser-tab.hh"
 
@@ -266,12 +267,9 @@ EvalState::EvalState(
                /nix/store while using a chroot store. */
             auto accessor = getFSSourceAccessor();
 
-            auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
-            if (settings.pureEval || store->storeDir != realStoreDir) {
-                accessor = settings.pureEval
-                    ? storeFS.cast<SourceAccessor>()
-                    : makeUnionSourceAccessor({accessor, storeFS});
-            }
+            accessor = settings.pureEval
+                ? storeFS.cast<SourceAccessor>()
+                : makeUnionSourceAccessor({accessor, storeFS});
 
             /* Apply access control if needed. */
             if (settings.restrictEval || settings.pureEval)
@@ -293,6 +291,7 @@ EvalState::EvalState(
     )}
     , store(store)
     , buildStore(buildStore ? buildStore : store)
+    , inputCache(fetchers::InputCache::create())
     , debugRepl(nullptr)
     , debugStop(false)
     , trylevel(0)
@@ -949,7 +948,16 @@ void EvalState::mkPos(Value & v, PosIdx p)
     auto origin = positions.originOf(p);
     if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(sFile).mkString(path->path.abs());
+        if (path->accessor == rootFS && store->isInStore(path->path.abs()))
+            // FIXME: only do this for virtual store paths?
+            attrs.alloc(sFile).mkString(path->path.abs(),
+                {
+                    NixStringContextElem::Opaque{
+                        .path = store->toStorePath(path->path.abs()).first
+                    }
+                });
+        else
+            attrs.alloc(sFile).mkString(path->path.abs());
         makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn));
         v.mkAttrs(attrs);
     } else
@@ -1135,6 +1143,7 @@ void EvalState::resetFileCache()
 {
     fileEvalCache.clear();
     fileParseCache.clear();
+    inputCache->clear();
 }
 
 
@@ -2317,6 +2326,9 @@ BackedStringView EvalState::coerceToString(
     }
 
     if (v.type() == nPath) {
+        // FIXME: instead of copying the path to the store, we could
+        // return a virtual store path that lazily copies the path to
+        // the store in devirtualize().
         return
             !canonicalizePath && !copyToStore
             ? // FIXME: hack to preserve path literals that end in a
@@ -2406,7 +2418,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
                 *store,
                 path.resolveSymlinks(SymlinkResolution::Ancestors),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
-                path.baseName(),
+                computeBaseName(path),
                 ContentAddressMethod::Raw::NixArchive,
                 nullptr,
                 repair);
@@ -2461,7 +2473,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (auto storePath = store->maybeParseStorePath(path))
         return *storePath;
-    error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
+    error<EvalError>("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path).withTrace(pos, errorCtx).debugThrow();
 }
 
 

src/libexpr/include/nix/expr/eval-settings.hh

@@ -247,6 +247,11 @@ struct EvalSettings : Config
 
           This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
         )"};
+
+    Setting<bool> lazyTrees{this, false, "lazy-trees",
+        R"(
+          If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily.
+        )"};
 };
 
 /**

src/libexpr/include/nix/expr/eval.hh

@@ -33,7 +33,11 @@ namespace nix {
 constexpr size_t maxPrimOpArity = 8;
 
 class Store;
-namespace fetchers { struct Settings; }
+namespace fetchers {
+struct Settings;
+struct InputCache;
+struct Input;
+}
 struct EvalSettings;
 class EvalState;
 class StorePath;
@@ -301,6 +305,8 @@ public:
 
     RootValue vImportedDrvToDerivation = nullptr;
 
+    ref<fetchers::InputCache> inputCache;
+
     /**
      * Debugger
      */
@@ -445,6 +451,15 @@ public:
 
     void checkURI(const std::string & uri);
 
+    /**
+     * Mount an input on the Nix store.
+     */
+    StorePath mountInput(
+        fetchers::Input & input,
+        const fetchers::Input & originalInput,
+        ref<SourceAccessor> accessor,
+        bool requireLockable);
+
     /**
      * Parse a Nix expression from the specified file.
      */
@@ -554,6 +569,18 @@ public:
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
         NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
 
+    StorePath devirtualize(
+        const StorePath & path,
+        StringMap * rewrites = nullptr);
+
+    SingleDerivedPath devirtualize(
+        const SingleDerivedPath & path,
+        StringMap * rewrites = nullptr);
+
+    std::string devirtualize(
+        std::string_view s,
+        const NixStringContext & context);
+
     /**
      * String coercion.
      *
@@ -569,6 +596,19 @@ public:
 
     StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
 
+
+    /**
+     * Compute the base name for a `SourcePath`. For non-store paths,
+     * this is just `SourcePath::baseName()`. But for store paths, for
+     * backwards compatibility, it needs to be `<hash>-source`,
+     * i.e. as if the path were copied to the Nix store. This results
+     * in a "double-copied" store path like
+     * `/nix/store/<hash1>-<hash2>-source`. We don't need to
+     * materialize /nix/store/<hash2>-source though. Still, this
+     * requires reading/hashing the path twice.
+     */
+    std::string computeBaseName(const SourcePath & path);
+
     /**
      * Path coercion.
      *

src/libexpr/include/nix/expr/print-ambiguous.hh

@@ -15,10 +15,10 @@ namespace nix {
  * See: https://github.com/NixOS/nix/issues/9730
  */
 void printAmbiguous(
-    Value &v,
-    const SymbolTable &symbols,
-    std::ostream &str,
-    std::set<const void *> *seen,
+    EvalState & state,
+    Value & v,
+    std::ostream & str,
+    std::set<const void *> * seen,
     int depth);
 
 }

src/libexpr/paths.cc

@@ -1,5 +1,7 @@
 #include "nix/store/store-api.hh"
 #include "nix/expr/eval.hh"
+#include "nix/util/mounted-source-accessor.hh"
+#include "nix/fetchers/fetch-to-store.hh"
 
 namespace nix {
 
@@ -18,4 +20,81 @@ SourcePath EvalState::storePath(const StorePath & path)
     return {rootFS, CanonPath{store->printStorePath(path)}};
 }
 
+StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites)
+{
+    if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) {
+        auto storePath = fetchToStore(
+            *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name());
+        assert(storePath.name() == path.name());
+        if (rewrites)
+            rewrites->emplace(path.hashPart(), storePath.hashPart());
+        return storePath;
+    } else
+        return path;
+}
+
+SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites)
+{
+    if (auto o = std::get_if<SingleDerivedPath::Opaque>(&path.raw()))
+        return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)};
+    else
+        return path;
+}
+
+std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context)
+{
+    StringMap rewrites;
+
+    for (auto & c : context)
+        if (auto o = std::get_if<NixStringContextElem::Opaque>(&c.raw))
+            devirtualize(o->path, &rewrites);
+
+    return rewriteStrings(std::string(s), rewrites);
+}
+
+std::string EvalState::computeBaseName(const SourcePath & path)
+{
+    if (path.accessor == rootFS) {
+        if (auto storePath = store->maybeParseStorePath(path.path.abs())) {
+            warn(
+                "Performing inefficient double copy of path '%s' to the store. "
+                "This can typically be avoided by rewriting an attribute like `src = ./.` "
+                "to `src = builtins.path { path = ./.; name = \"source\"; }`.",
+                path);
+            return std::string(fetchToStore(*store, path, FetchMode::DryRun, storePath->name()).to_string());
+        }
+    }
+    return std::string(path.baseName());
+}
+
+StorePath EvalState::mountInput(
+    fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor, bool requireLockable)
+{
+    auto storePath = settings.lazyTrees ? StorePath::random(input.getName())
+                                        : fetchToStore(*store, accessor, FetchMode::Copy, input.getName());
+
+    allowPath(storePath); // FIXME: should just whitelist the entire virtual store
+
+    storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
+
+    if (requireLockable && !input.isLocked() && !input.getNarHash()) {
+        auto narHash = accessor->hashPath(CanonPath::root);
+        input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
+    }
+
+    // FIXME: what to do with the NAR hash in lazy mode?
+    if (!settings.lazyTrees && originalInput.getNarHash()) {
+        auto expected = originalInput.computeStorePath(*store);
+        if (storePath != expected)
+            throw Error(
+                (unsigned int) 102,
+                "NAR hash mismatch in input '%s', expected '%s' but got '%s'",
+                originalInput.to_string(),
+                store->printStorePath(storePath),
+                store->printStorePath(expected));
+    }
+
+    return storePath;
+}
+
 }

src/libexpr/primops.cc

@@ -14,6 +14,7 @@
 #include "nix/expr/value-to-xml.hh"
 #include "nix/expr/primops.hh"
 #include "nix/fetchers/fetch-to-store.hh"
+#include "nix/util/mounted-source-accessor.hh"
 
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
@@ -75,7 +76,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
                 ensureValid(b.drvPath->getBaseStorePath());
             },
             [&](const NixStringContextElem::Opaque & o) {
-                ensureValid(o.path);
+                // We consider virtual store paths valid here. They'll
+                // be devirtualized if needed elsewhere.
+                if (!storeFS->getMount(CanonPath(store->printStorePath(o.path))))
+                    ensureValid(o.path);
                 if (maybePathsOut)
                     maybePathsOut->emplace(o.path);
             },
@@ -1408,6 +1412,8 @@ static void derivationStrictInternal(
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
+    StringMap rewrites;
+
     for (auto & c : context) {
         std::visit(overloaded {
             /* Since this allows the builder to gain access to every
@@ -1430,11 +1436,13 @@ static void derivationStrictInternal(
                 drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
             },
             [&](const NixStringContextElem::Opaque & o) {
-                drv.inputSrcs.insert(o.path);
+                drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites));
             },
         }, c.raw);
     }
 
+    drv.applyRewrites(rewrites);
+
     /* Do we have all required attributes? */
     if (drv.builder == "")
         state.error<EvalError>("required attribute 'builder' missing")
@@ -2500,6 +2508,7 @@ static void addPath(
                 {}));
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+            // FIXME: make this lazy?
             auto dstPath = fetchToStore(
                 *state.store,
                 path.resolveSymlinks(),
@@ -2530,7 +2539,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
         "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
 
-    addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
+    addPath(state, pos, state.computeBaseName(path), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({