diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e404538a1a2c..f8927a3448e7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1739,6 +1739,7 @@ static void derivationStrictInternal( drv.structuredAttrs = std::move(*jsonObject); } + /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ @@ -1877,6 +1878,26 @@ static void derivationStrictInternal( drv.fillInOutputPaths(*state.store); } + /* Override output paths for builtin:fetch-closure */ + if (isImpure && drv.builder == "builtin:fetch-closure" && drv.structuredAttrs) { + auto & structuredAttrs = drv.structuredAttrs->structuredAttrs; + auto fromPathIt = structuredAttrs.find("fromPath"); + if (fromPathIt != structuredAttrs.end() && fromPathIt->second.is_string()) { + auto toPathIt = structuredAttrs.find("toPath"); + auto pathStr = (toPathIt != structuredAttrs.end() && toPathIt->second.is_string()) + ? toPathIt->second.get() + : fromPathIt->second.get(); + + auto lastSlash = pathStr.rfind('/'); + StorePath outputPath(lastSlash != std::string::npos ? pathStr.substr(lastSlash + 1) : pathStr); + + for (auto & [outputName, _] : drv.outputs) { + drv.env[outputName] = state.store->printStorePath(outputPath); + drv.outputs.insert_or_assign(outputName, DerivationOutput::InputAddressed{.path = outputPath}); + } + } + } + /* Write the resulting term into the Nix store directory. */ auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair, false, provenance); auto drvPathS = state.store->printStorePath(drvPath); diff --git a/src/libstore/builtins/fetch-closure.cc b/src/libstore/builtins/fetch-closure.cc new file mode 100644 index 000000000000..eef8b268f63d --- /dev/null +++ b/src/libstore/builtins/fetch-closure.cc @@ -0,0 +1,196 @@ +#include "nix/store/builtins.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/store/store-open.hh" +#include "nix/store/realisation.hh" +#include "nix/store/make-content-addressed.hh" +#include "nix/store/nar-info-disk-cache.hh" +#include "nix/store/nar-info.hh" +#include "nix/store/filetransfer.hh" +#include "nix/store/globals.hh" +#include "nix/store/store-dir-config.hh" +#include "nix/util/url.hh" +#include "nix/util/archive.hh" +#include "nix/util/compression.hh" +#include "nix/util/file-system.hh" +#include "nix/util/util.hh" +#include "nix/util/environment-variables.hh" + +#include +#include + +namespace nix { + +static void builtinFetchClosure(const BuiltinBuilderContext & ctx) +{ + experimentalFeatureSettings.require(Xp::FetchClosure); + + auto out = get(ctx.drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetch-closure' requires an 'out' output"); + + if (!ctx.drv.structuredAttrs) + throw Error("'builtin:fetch-closure' must have '__structuredAttrs = true'"); + + auto & attrs = ctx.drv.structuredAttrs->structuredAttrs; + + // Parse attributes + std::optional fromStoreUrl; + std::optional fromPath; + std::optional toPath; + bool inputAddressed = false; + + if (auto it = attrs.find("fromStore"); it != attrs.end() && it->second.is_string()) { + fromStoreUrl = it->second.get(); + } + + if (auto it = attrs.find("fromPath"); it != attrs.end() && it->second.is_string()) { + auto pathStr = it->second.get(); + // Extract basename if full path provided + auto lastSlash = pathStr.rfind('/'); + fromPath = StorePath(lastSlash != std::string::npos ? pathStr.substr(lastSlash + 1) : pathStr); + } + + if (auto it = attrs.find("toPath"); it != attrs.end() && it->second.is_string()) { + auto pathStr = it->second.get(); + // Extract basename if full path provided + auto lastSlash = pathStr.rfind('/'); + toPath = StorePath(lastSlash != std::string::npos ? pathStr.substr(lastSlash + 1) : pathStr); + } + + if (auto it = attrs.find("inputAddressed"); it != attrs.end() && it->second.is_boolean()) { + inputAddressed = it->second.get(); + } + + if (!fromStoreUrl) + throw Error("'builtin:fetch-closure' requires 'fromStore' attribute"); + + if (!fromPath) + throw Error("'builtin:fetch-closure' requires 'fromPath' attribute"); + + // Validate URL + auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true); + + if (parsedURL.scheme != "http" && parsedURL.scheme != "https" + && !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) + throw Error("'builtin:fetch-closure' only supports http:// and https:// stores"); + + if (!parsedURL.query.empty()) + throw Error("'builtin:fetch-closure' does not support URL query parameters (in '%s')", *fromStoreUrl); + + std::cerr << fmt("fetching closure '%s' from '%s'...\n", + fromPath->to_string(), *fromStoreUrl); + + // Open the remote store to get its storeDir + auto fromStore = openStore(*fromStoreUrl); + auto remoteStoreDir = fromStore->storeDir; + + // Extract local store directory from derivation output path + auto derivationOutputPath = ctx.outputs.at("out"); + auto lastSlash = derivationOutputPath.rfind('/'); + if (lastSlash == std::string::npos) + throw Error("invalid output path '%s'", derivationOutputPath); + Path localStoreDir = derivationOutputPath.substr(0, lastSlash); + + // Verify store directories match + if (remoteStoreDir != localStoreDir) + throw Error("store directory mismatch: remote store uses '%s' but local store uses '%s'", + remoteStoreDir, localStoreDir); + + // Create a fresh FileTransfer since we're in a forked process + auto fileTransfer = makeFileTransfer(); + + // Download and parse .narinfo to get metadata + auto narInfoUrl = parsedURL.to_string(); + if (!hasSuffix(narInfoUrl, "/")) narInfoUrl += "/"; + narInfoUrl += fromPath->hashPart() + ".narinfo"; + + StoreDirConfig storeDirConfig{.storeDir = remoteStoreDir}; + + std::shared_ptr narInfo; + try { + FileTransferRequest request(VerbatimURL{narInfoUrl}); + auto result = fileTransfer->download(request); + narInfo = std::make_shared(storeDirConfig, result.data, narInfoUrl); + + // Verify the path matches + if (narInfo->path != *fromPath) + throw Error("NAR info path mismatch: expected '%s', got '%s'", + fromPath->to_string(), narInfo->path.to_string()); + } catch (FileTransferError & e) { + throw Error("failed to fetch NAR info for '%s' from '%s': %s", + fromPath->to_string(), *fromStoreUrl, e.what()); + } + + // Validate content-addressed vs input-addressed + bool isCA = narInfo->ca.has_value(); + + if (inputAddressed && isCA) + throw Error("The store object at '%s' is content-addressed, but 'inputAddressed' is set to 'true'", + fromPath->to_string()); + + if (!inputAddressed && !isCA) + throw Error("The store object at '%s' is input-addressed, but 'inputAddressed' is not set.\n\n" + "Add 'inputAddressed = true;' if you intend to fetch an input-addressed store path.", + fromPath->to_string()); + + // Derive the output path from fromPath (or toPath if rewriting) + auto outputStorePath = toPath ? *toPath : *fromPath; + std::string outputPath = localStoreDir + "/" + outputStorePath.to_string(); + + // Download and unpack NAR + auto narUrl = parsedURL.to_string(); + if (!hasSuffix(narUrl, "/")) narUrl += "/"; + narUrl += narInfo->url; + + // If rewriting, we need to unpack to a temporary location first + Path unpackPath = outputPath; + std::optional tempPath; + + if (toPath) { + // Create temporary directory for unpacking + tempPath = outputPath + ".tmp"; + unpackPath = *tempPath; + } + + auto source = sinkToSource([&](Sink & sink) { + auto decompressor = makeDecompressionSink(narInfo->compression, sink); + FileTransferRequest request(VerbatimURL{narUrl}); + fileTransfer->download(std::move(request), *decompressor); + decompressor->finish(); + }); + + restorePath(unpackPath, *source); + + // Rewrite store path references if toPath is provided + if (toPath) { + std::cerr << fmt("rewriting references from '%s' to '%s'...\n", + fromPath->to_string(), toPath->to_string()); + + StringMap rewrites; + rewrites[std::string(fromPath->to_string())] = std::string(toPath->to_string()); + + // Recursively rewrite all files + std::function rewritePath = [&](const Path & path) { + for (auto & entry : std::filesystem::directory_iterator(path)) { + auto entryPath = entry.path().string(); + if (entry.is_directory() && !entry.is_symlink()) { + rewritePath(entryPath); + } else if (entry.is_regular_file()) { + auto content = readFile(entryPath); + auto newContent = rewriteStrings(content, rewrites); + if (newContent != content) + writeFile(entryPath, newContent); + } + } + }; + + rewritePath(unpackPath); + + // Move to final output path + std::filesystem::rename(unpackPath, outputPath); + } +} + +static RegisterBuiltinBuilder registerFetchClosure("fetch-closure", builtinFetchClosure); + +} // namespace nix diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 5994e7cb43eb..68427fda1bb4 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1313,6 +1313,11 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string if (outputVariant.path == outPath) { return; // Correct case } + /* Special case: builtin:fetch-closure can have arbitrary output paths */ + if (drv.isBuiltin() && drv.builder == "builtin:fetch-closure") { + envHasRightPath(outputVariant.path); + return; + } /* Error case, an explicitly wrong path is always an error. */ throw Error( diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 7a53fd65d86a..5aa4f683f435 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -305,6 +305,7 @@ sources = files( 'build/substitution-goal.cc', 'build/worker.cc', 'builtins/buildenv.cc', + 'builtins/fetch-closure.cc', 'builtins/fetchurl.cc', 'builtins/unpack-channel.cc', 'common-protocol.cc', diff --git a/tests/functional/builtin-fetch-closure.sh b/tests/functional/builtin-fetch-closure.sh new file mode 100755 index 000000000000..4937382bda2e --- /dev/null +++ b/tests/functional/builtin-fetch-closure.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +source common.sh + +enableFeatures "fetch-closure impure-derivations" + +TODO_NixOS + +clearStore +clearCacheCache + +# Old daemons don't properly zero out the self-references when +# calculating the CA hashes, so this breaks `nix store +# make-content-addressed` which expects the client and the daemon to +# compute the same hash +requireDaemonNewerThan "2.16.0pre20230524" + +# Initialize binary cache. +nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) +caPath=$(nix store make-content-addressed --json "$nonCaPath" | jq -r '.rewrites | map(.) | .[]') +nix copy --to file://"$cacheDir" "$nonCaPath" +nix copy --to file://"$cacheDir" "$caPath" + +# Test basic builtin:fetch-closure with input-addressed path +clearStore + +[ ! -e "$nonCaPath" ] + +outPath=$(nix-build --no-out-link --expr " + derivation { + name = \"fetch-test\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __impure = true; + __structuredAttrs = true; + fromStore = \"file://$cacheDir\"; + fromPath = \"$nonCaPath\"; + inputAddressed = true; + } +") + +echo "outPath = $outPath" + +[ "$outPath" = "$nonCaPath" ] +[ -e "$nonCaPath" ] + +clearStore + +# Test builtin:fetch-closure with CA path +clearStore + +[ ! -e "$caPath" ] + +outPath=$(nix-build --no-out-link --expr " + derivation { + name = \"fetch-test-ca\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __impure = true; + __structuredAttrs = true; + fromStore = \"file://$cacheDir\"; + fromPath = \"$caPath\"; + } +") + +echo "outPath = $outPath" + +[ "$outPath" = "$caPath" ] +[ -e "$caPath" ] + +clearStore + +# Test builtin:fetch-closure with full path +clearStore + +[ ! -e "$nonCaPath" ] + +outPath=$(nix-build --no-out-link --expr " + derivation { + name = \"fetch-test-fullpath\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __impure = true; + __structuredAttrs = true; + fromStore = \"file://$cacheDir\"; + fromPath = \"/nix/store/$(basename $nonCaPath)\"; + inputAddressed = true; + } +") + +echo "outPath = $outPath" + +[ "$outPath" = "$nonCaPath" ] +[ -e "$nonCaPath" ] + +clearStore + +# Test that missing __structuredAttrs fails +expectStderr 1 nix-build --no-out-link --expr " + derivation { + name = \"fetch-test-nostruct\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __impure = true; + fromStore = \"file://$cacheDir\"; + fromPath = \"$nonCaPath\"; + inputAddressed = true; + } +" + +# Test that missing __impure fails (derivation won't have predetermined path) +expectStderr 1 nix-build --no-out-link --expr " + derivation { + name = \"fetch-test-noimpure\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __structuredAttrs = true; + fromStore = \"file://$cacheDir\"; + fromPath = \"$nonCaPath\"; + inputAddressed = true; + } +" + +# Test that URL query parameters aren't allowed +expectStderr 100 nix-build --no-out-link --expr " + derivation { + name = \"fetch-test-query\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __impure = true; + __structuredAttrs = true; + fromStore = \"file://$cacheDir?foo=bar\"; + fromPath = \"$nonCaPath\"; + inputAddressed = true; + } +" | grepQuiet "does not support URL query parameters" + +# Test CA/input-addressed mismatch detection +expectStderr 100 nix-build --no-out-link --expr " + derivation { + name = \"fetch-test-mismatch\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __impure = true; + __structuredAttrs = true; + fromStore = \"file://$cacheDir\"; + fromPath = \"$caPath\"; + inputAddressed = true; + } +" | grepQuiet "is content-addressed, but 'inputAddressed' is set to 'true'" + +expectStderr 100 nix-build --no-out-link --expr " + derivation { + name = \"fetch-test-mismatch2\"; + builder = \"builtin:fetch-closure\"; + system = \"$system\"; + __impure = true; + __structuredAttrs = true; + fromStore = \"file://$cacheDir\"; + fromPath = \"$nonCaPath\"; + } +" | grepQuiet "is input-addressed, but 'inputAddressed' is not set" diff --git a/tests/functional/meson.build b/tests/functional/meson.build index d917d91c3f34..ccc546038616 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -161,6 +161,7 @@ suites = [ 'suggestions.sh', 'store-info.sh', 'fetchClosure.sh', + 'builtin-fetch-closure.sh', 'completions.sh', 'impure-derivations.sh', 'path-from-hash-part.sh',