diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 4b58ea16d05e..7c5c5be1036f 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -58,7 +58,7 @@ std::pair fetchToStore2( } } else { static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; - if (barf && !filter) + if (barf && !filter && !(path.to_string().starts_with("/") || path.to_string().starts_with("«path:/"))) throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); // FIXME: could still provide in-memory caching keyed on `SourcePath`. debug("source path '%s' is uncacheable", path); diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index f883c0921903..6b0748860c80 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -68,6 +68,11 @@ std::pair> FilteringSourceAccessor::getFin return next->getFingerprint(prefix / path); } +void FilteringSourceAccessor::invalidateCache(const CanonPath & path) +{ + next->invalidateCache(prefix / path); +} + void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 5e98caa58165..63df495907a5 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -52,6 +52,8 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; + void invalidateCache(const CanonPath & path) override; + /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 2f2d6976a14f..8f9ccd19132a 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -142,41 +142,27 @@ struct PathInputScheme : InputScheme getAccessor(const Settings & settings, Store & store, const Input & _input) const override { Input input(_input); - auto path = getStrAttr(input.attrs, "path"); auto absPath = getAbsPath(input); // FIXME: check whether access to 'path' is allowed. + + auto accessor = makeFSSourceAccessor(absPath); + auto storePath = store.maybeParseStorePath(absPath.string()); - if (storePath) + if (storePath) { store.addTempRoot(*storePath); - time_t mtime = 0; - if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); - // FIXME: try to substitute storePath. - auto src = sinkToSource( - [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); - storePath = store.addToStoreFromDump(*src, "source"); + // To prevent `fetchToStore()` copying the path again to Nix + // store, pre-create an entry in the fetcher cache. + auto info = store.queryPathInfo(*storePath); + accessor->fingerprint = fmt("path:%s", info->narHash.to_string(HashFormat::SRI, true)); + settings.getCache()->upsert( + makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + {{"hash", info->narHash.to_string(HashFormat::SRI, true)}}); } - auto accessor = store.requireStoreObjectAccessor(*storePath); - - // To prevent `fetchToStore()` copying the path again to Nix - // store, pre-create an entry in the fetcher cache. - auto info = store.queryPathInfo(*storePath); - accessor->fingerprint = - fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); - settings.getCache()->upsert( - makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), - {{"hash", info->narHash.to_string(HashFormat::SRI, true)}}); - - /* Trust the lastModified value supplied by the user, if - any. It's not a "secure" attribute so we don't care. */ - if (!input.getLastModified()) - input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {accessor, std::move(input)}; } }; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 0e7760b62494..66d18f09f1aa 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -436,7 +436,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; - auto flake = getFlake(state, topRef, useRegistriesTop, {}, lockFlags.requireLockable); + auto flake = getFlake(state, topRef, useRegistriesTop, {}, false); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -919,6 +919,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), newLockFileS, commitMessage); + + flake.lockFilePath().invalidateCache(); } /* Rewriting the lockfile changed the top-level diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index 29561a3daafe..006ba0e7e4d1 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -78,6 +78,8 @@ public: return trackLastModified ? std::optional{mtime} : std::nullopt; } + void invalidateCache(const CanonPath & path) override; + private: /** diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 1006895b33c0..1357cf79a285 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -209,6 +209,11 @@ struct SourceAccessor : std::enable_shared_from_this { return std::nullopt; } + + /** + * Invalidate any cached value the accessor may have for the specified path. + */ + virtual void invalidateCache(const CanonPath & path) {} }; /** diff --git a/src/libutil/include/nix/util/source-path.hh b/src/libutil/include/nix/util/source-path.hh index 08f9fe580b03..4597de1ff46e 100644 --- a/src/libutil/include/nix/util/source-path.hh +++ b/src/libutil/include/nix/util/source-path.hh @@ -114,6 +114,11 @@ struct SourcePath return {accessor, accessor->resolveSymlinks(path, mode)}; } + void invalidateCache() const + { + accessor->invalidateCache(path); + } + friend class std::hash; }; diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index d9398045cc56..13b77d2d1e1e 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -99,6 +99,12 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor auto [accessor, subpath] = resolve(path); return accessor->getFingerprint(subpath); } + + void invalidateCache(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + accessor->invalidateCache(subpath); + } }; ref makeMountedSourceAccessor(std::map> mounts) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index abbab45db21c..632504e74a05 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -87,11 +87,11 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) return nix::pathExists(makeAbsPath(path).string()); } +using Cache = boost::concurrent_flat_map>; +static Cache cache; + std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) { - using Cache = boost::concurrent_flat_map>; - static Cache cache; - // Note: we convert std::filesystem::path to Path because the // former is not hashable on libc++. Path absPath = makeAbsPath(path).string(); @@ -108,6 +108,11 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa return st; } +void PosixSourceAccessor::invalidateCache(const CanonPath & path) +{ + cache.erase(makeAbsPath(path).string()); +} + std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index e3b39f14ed27..f0a72116447e 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -84,6 +84,12 @@ struct UnionSourceAccessor : SourceAccessor } return {path, std::nullopt}; } + + void invalidateCache(const CanonPath & path) override + { + for (auto & accessor : accessors) + accessor->invalidateCache(path); + } }; ref makeUnionSourceAccessor(std::vector> && accessors) diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index fd59385762de..3e3aeef3ddcf 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -361,4 +361,25 @@ execUnshare () { exec unshare --mount --map-root-user "$SHELL" "$@" } +initGitRepo() { + local repo="$1" + local extraArgs="${2-}" + + # shellcheck disable=SC2086 # word splitting of extraArgs is intended + git -C "$repo" init $extraArgs + git -C "$repo" config user.email "foobar@example.com" + git -C "$repo" config user.name "Foobar" +} + +createGitRepo() { + local repo="$1" + local extraArgs="${2-}" + + rm -rf "$repo" "$repo".tmp + mkdir -p "$repo" + + # shellcheck disable=SC2086 # word splitting of extraArgs is intended + initGitRepo "$repo" $extraArgs +} + fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 31a8d0d2bc14..e6d47e7f33ec 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -12,11 +12,9 @@ repo=$TEST_ROOT/./git export _NIX_FORCE_HTTP=1 -rm -rf "$repo" "${repo}"-tmp "$TEST_HOME"/.cache/nix "$TEST_ROOT"/worktree "$TEST_ROOT"/minimal +rm -rf "${repo}"-tmp "$TEST_HOME"/.cache/nix "$TEST_ROOT"/worktree "$TEST_ROOT"/minimal -git init "$repo" -git -C "$repo" config user.email "foobar@example.com" -git -C "$repo" config user.name "Foobar" +createGitRepo "$repo" echo utrecht > "$repo"/hello touch "$repo"/.gitignore @@ -213,8 +211,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = # Fetching from a repo with only a specific revision and no branches should # not fall back to copying files and record correct revision information. See: #5302 -mkdir "$TEST_ROOT"/minimal -git -C "$TEST_ROOT"/minimal init +createGitRepo "$TEST_ROOT"/minimal git -C "$TEST_ROOT"/minimal fetch "$repo" "$rev2" git -C "$TEST_ROOT"/minimal checkout "$rev2" [[ $(nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/minimal; }).rev") = "$rev2" ]] @@ -267,7 +264,7 @@ rm -rf "$TEST_HOME"/.cache/nix (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # should succeed for a repo without commits -git init "$repo" +initGitRepo "$repo" git -C "$repo" add hello # need to add at least one file to cause the root of the repo to be visible # shellcheck disable=SC2034 path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") @@ -275,9 +272,7 @@ path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").ou # should succeed for a path with a space # regression test for #7707 repo="$TEST_ROOT/a b" -git init "$repo" -git -C "$repo" config user.email "foobar@example.com" -git -C "$repo" config user.name "Foobar" +createGitRepo "$repo" echo utrecht > "$repo/hello" touch "$repo/.gitignore" @@ -289,7 +284,7 @@ path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") # Test a workdir with no commits. empty="$TEST_ROOT/empty" -git init "$empty" +createGitRepo "$empty" emptyAttrs="{ lastModified = 0; lastModifiedDate = \"19700101000000\"; narHash = \"sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo=\"; rev = \"0000000000000000000000000000000000000000\"; revCount = 0; shortRev = \"0000000\"; submodules = false; }" result=$(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") @@ -317,10 +312,7 @@ nix eval --impure --expr "let attrs = builtins.fetchGit $empty; in assert attrs. # Test backward compatibility hack for Nix < 2.20 locks / fetchTree calls that expect Git filters to be applied. eol="$TEST_ROOT/git-eol" -mkdir -p "$eol" -git init "$eol" -git -C "$eol" config user.email "foobar@example.com" -git -C "$eol" config user.name "Foobar" +createGitRepo "$eol" printf "Hello\nWorld\n" > "$eol/crlf" printf "ignore me" > "$eol/ignored" git -C "$eol" add crlf ignored diff --git a/tests/functional/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh index a7d1a2a2931c..9c7fb323eb7e 100755 --- a/tests/functional/fetchGitRefs.sh +++ b/tests/functional/fetchGitRefs.sh @@ -8,11 +8,9 @@ clearStoreIfPossible repo="$TEST_ROOT/git" -rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix" +rm -rf "${repo}-tmp" "$TEST_HOME/.cache/nix" -git init "$repo" -git -C "$repo" config user.email "foobar@example.com" -git -C "$repo" config user.name "Foobar" +createGitRepo "$repo" echo utrecht > "$repo/hello" git -C "$repo" add hello diff --git a/tests/functional/fetchGitShallow.sh b/tests/functional/fetchGitShallow.sh index 4c21bd7ac80a..6b91d60cd9e3 100644 --- a/tests/functional/fetchGitShallow.sh +++ b/tests/functional/fetchGitShallow.sh @@ -6,9 +6,7 @@ source common.sh requireGit # Create a test repo with multiple commits for all our tests -git init "$TEST_ROOT/shallow-parent" -git -C "$TEST_ROOT/shallow-parent" config user.email "foobar@example.com" -git -C "$TEST_ROOT/shallow-parent" config user.name "Foobar" +createGitRepo "$TEST_ROOT/shallow-parent" # Add several commits to have history echo "{ outputs = _: {}; }" > "$TEST_ROOT/shallow-parent/flake.nix" diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index 2a25245be756..bf5fe5df3877 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -22,22 +22,16 @@ rm -rf "${rootRepo}" "${subRepo}" "$TEST_HOME"/.cache/nix export XDG_CONFIG_HOME=$TEST_HOME/.config git config --global protocol.file.allow always -initGitRepo() { - git init "$1" - git -C "$1" config user.email "foobar@example.com" - git -C "$1" config user.name "Foobar" -} - addGitContent() { echo "lorem ipsum" > "$1"/content git -C "$1" add content git -C "$1" commit -m "Initial commit" } -initGitRepo "$subRepo" +createGitRepo "$subRepo" addGitContent "$subRepo" -initGitRepo "$rootRepo" +createGitRepo "$rootRepo" git -C "$rootRepo" submodule init git -C "$rootRepo" submodule add "$subRepo" sub @@ -199,19 +193,19 @@ test_submodule_nested() { local repoB=$TEST_ROOT/submodule_nested/b local repoC=$TEST_ROOT/submodule_nested/c - rm -rf "$repoA" "$repoB" "$repoC" "$TEST_HOME"/.cache/nix + rm -rf "$TEST_HOME"/.cache/nix - initGitRepo "$repoC" + createGitRepo "$repoC" touch "$repoC"/inside-c git -C "$repoC" add inside-c addGitContent "$repoC" - initGitRepo "$repoB" + createGitRepo "$repoB" git -C "$repoB" submodule add "$repoC" c git -C "$repoB" add c addGitContent "$repoB" - initGitRepo "$repoA" + createGitRepo "$repoA" git -C "$repoA" submodule add "$repoB" b git -C "$repoA" add b addGitContent "$repoA" diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh index 79c78d0c9f6f..3b5f9b866b9a 100755 --- a/tests/functional/fetchGitVerification.sh +++ b/tests/functional/fetchGitVerification.sh @@ -21,9 +21,7 @@ ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" key2File="$keysDir/testkey2.pub" publicKey2=$(awk '{print $2}' "$key2File") -git init "$repo" -git -C "$repo" config user.email "foobar@example.com" -git -C "$repo" config user.name "Foobar" +createGitRepo "$repo" git -C "$repo" config gpg.format ssh echo 'hello' > "$repo"/text diff --git a/tests/functional/fetchPath.sh b/tests/functional/fetchPath.sh index 1df895b61662..2784afb0388e 100755 --- a/tests/functional/fetchPath.sh +++ b/tests/functional/fetchPath.sh @@ -3,9 +3,9 @@ source common.sh touch "$TEST_ROOT/foo" -t 202211111111 -# We only check whether 2022-11-1* **:**:** is the last modified date since -# `lastModified` is transformed into UTC in `builtins.fetchTarball`. -[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] + +# The path fetcher does not return lastModified. +[[ "$(nix eval --impure --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\") ? lastModifiedDate")" = false ]] # Check that we can override lastModified for "path:" inputs. [[ "$(nix eval --impure --expr "(builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; lastModified = 123; }).lastModified")" = 123 ]] diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 1c7c5664da05..6fef78925592 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -115,24 +115,3 @@ writeTrivialFlake() { } EOF } - -initGitRepo() { - local repo="$1" - local extraArgs="${2-}" - - # shellcheck disable=SC2086 # word splitting of extraArgs is intended - git -C "$repo" init $extraArgs - git -C "$repo" config user.email "foobar@example.com" - git -C "$repo" config user.name "Foobar" -} - -createGitRepo() { - local repo="$1" - local extraArgs="${2-}" - - rm -rf "$repo" "$repo".tmp - mkdir -p "$repo" - - # shellcheck disable=SC2086 # word splitting of extraArgs is intended - initGitRepo "$repo" $extraArgs -} diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 0efd6b125a17..c33b2a64ae1d 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -381,7 +381,6 @@ nix build -o "$TEST_ROOT"/result git+file://"$flakeGitBare" mkdir -p "$flake5Dir" writeDependentFlake "$flake5Dir" nix flake lock path://"$flake5Dir" -[[ "$(nix flake metadata "path://$flake5Dir" --json | jq -r .fingerprint)" != null ]] # Test tarball flakes. tar cfz "$TEST_ROOT"/flake.tar.gz -C "$TEST_ROOT" flake5 diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 7480cd504584..323e97ba9cc0 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -135,10 +135,8 @@ EOF # https://github.com/NixOS/nix/issues/13164 mkdir -p "$TEST_ROOT/issue-13164/nested-flake1/nested-flake2" ( + initGitRepo "$TEST_ROOT/issue-13164" cd "$TEST_ROOT/issue-13164" - git init - git config --global user.email "you@example.com" - git config --global user.name "Your Name" cat >flake.nix < "$flake1Dir"/flake.nix < "$flake1Dir"/ca.nix cp "${config_nix}" "$flake1Dir"/ +git -C "$flake1Dir" add flake.nix config.nix who version ca.nix +git -C "$flake1Dir" commit -m 'Initial' + # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0' nix profile add "$flake1Dir" -L -nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash' +#nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash' [[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World" ]] [ -e "$TEST_HOME"/.nix-profile/share/man ] # shellcheck disable=SC2235 @@ -224,11 +229,11 @@ error: An existing package already provides the following file: The conflicting packages have a priority of 5. To prioritise the new package: - nix profile add path:${flake2Dir}#packages.${system}.default --priority 4 + nix profile add git+file://${flake2Dir}#packages.${system}.default --priority 4 To prioritise the existing package: - nix profile add path:${flake2Dir}#packages.${system}.default --priority 6 + nix profile add git+file://${flake2Dir}#packages.${system}.default --priority 6 EOF ) [[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World" ]]