Skip to content

Commit 779d7a9

Browse files
edolstraedolstra
committed
Factor out the local NAR cache into its own class
1 parent 09d8dd7 commit 779d7a9

8 files changed

Lines changed: 158 additions & 93 deletions

File tree

‎src/libstore/binary-cache-store.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "nix/util/callback.hh"
1414
#include "nix/util/signals.hh"
1515
#include "nix/util/archive.hh"
16+
#include "nix/store/nar-cache.hh"
1617

1718
#include <chrono>
1819
#include <future>
@@ -26,6 +27,7 @@ namespace nix {
2627

2728
BinaryCacheStore::BinaryCacheStore(Config & config)
2829
: config{config}
30+
, narCache{config.localNarCache.get().empty() ? nullptr : std::make_shared<NarCache>(config.localNarCache.get())}
2931
{
3032
if (config.secretKeyFile != "")
3133
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
@@ -541,7 +543,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
541543

542544
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
543545
{
544-
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
546+
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, narCache);
545547
}
546548

547549
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)

‎src/libstore/include/nix/store/binary-cache-store.hh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace nix {
1313

1414
struct NarInfo;
15+
class NarCache;
1516
class RemoteFSAccessor;
1617

1718
struct BinaryCacheStoreConfig : virtual StoreConfig
@@ -85,6 +86,8 @@ protected:
8586

8687
const std::string cacheInfoFile = "nix-cache-info";
8788

89+
std::shared_ptr<NarCache> narCache;
90+
8891
BinaryCacheStore(Config &);
8992

9093
public:

‎src/libstore/include/nix/store/meson.build‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ headers = [ config_pub_h ] + files(
5656
'make-content-addressed.hh',
5757
'names.hh',
5858
'nar-accessor.hh',
59+
'nar-cache.hh',
5960
'nar-info-disk-cache.hh',
6061
'nar-info.hh',
6162
'outputs-spec.hh',

‎src/libstore/include/nix/store/nar-cache.hh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#pragma once
2+
3+
#include "nix/util/hash.hh"
4+
5+
#include <filesystem>
6+
7+
namespace nix {
8+
9+
class NarCache
10+
{
11+
12+
const std::filesystem::path cacheDir;
13+
14+
std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
15+
16+
public:
17+
18+
NarCache(std::filesystem::path cacheDir);
19+
20+
void upsertNar(const Hash & narHash, Source & source);
21+
22+
void upsertNarListing(const Hash & narHash, std::string_view narListingData);
23+
24+
// FIXME: use a sink.
25+
std::optional<std::string> getNar(const Hash & narHash);
26+
27+
// FIXME: use a sink.
28+
std::string getNarBytes(const Hash & narHash, uint64_t offset, uint64_t length);
29+
30+
std::optional<std::string> getNarListing(const Hash & narHash);
31+
};
32+
33+
} // namespace nix

‎src/libstore/include/nix/store/remote-fs-accessor.hh‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
namespace nix {
99

10+
struct NarCache;
11+
1012
class RemoteFSAccessor : public SourceAccessor
1113
{
1214
ref<Store> store;
@@ -15,28 +17,20 @@ class RemoteFSAccessor : public SourceAccessor
1517

1618
bool requireValidPath;
1719

18-
std::filesystem::path cacheDir;
20+
std::shared_ptr<NarCache> narCache;
1921

2022
std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
2123

2224
friend struct BinaryCacheStore;
2325

24-
std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
25-
26-
ref<SourceAccessor> addToCache(
27-
std::string_view hashPart,
28-
const std::filesystem::path & cacheFile,
29-
const std::filesystem::path & listingFile,
30-
std::string && nar);
31-
3226
public:
3327

3428
/**
3529
* @return nullptr if the store does not contain any object at that path.
3630
*/
3731
std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
3832

39-
RemoteFSAccessor(ref<Store> store, bool requireValidPath = true, std::filesystem::path cacheDir = {});
33+
RemoteFSAccessor(ref<Store> store, bool requireValidPath = true, std::shared_ptr<NarCache> narCache = {});
4034

4135
std::optional<Stat> maybeLstat(const CanonPath & path) override;
4236

‎src/libstore/meson.build‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,7 @@ sources = files(
345345
'misc.cc',
346346
'names.cc',
347347
'nar-accessor.cc',
348+
'nar-cache.cc',
348349
'nar-info-disk-cache.cc',
349350
'nar-info.cc',
350351
'optimise-store.cc',

‎src/libstore/nar-cache.cc‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#include "nix/store/nar-cache.hh"
2+
#include "nix/util/file-system.hh"
3+
4+
#include <sys/types.h>
5+
#include <sys/stat.h>
6+
#include <fcntl.h>
7+
8+
namespace nix {
9+
10+
NarCache::NarCache(std::filesystem::path cacheDir_)
11+
: cacheDir(std::move(cacheDir_))
12+
{
13+
assert(!cacheDir.empty());
14+
createDirs(cacheDir);
15+
}
16+
17+
std::filesystem::path NarCache::makeCacheFile(const Hash & narHash, const std::string & ext)
18+
{
19+
return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
20+
}
21+
22+
void NarCache::upsertNar(const Hash & narHash, Source & source)
23+
{
24+
try {
25+
/* FIXME: do this asynchronously. */
26+
writeFile(makeCacheFile(narHash, "nar"), source);
27+
} catch (SystemError &) {
28+
ignoreExceptionExceptInterrupt();
29+
}
30+
}
31+
32+
void NarCache::upsertNarListing(const Hash & narHash, std::string_view narListingData)
33+
{
34+
try {
35+
writeFile(makeCacheFile(narHash, "ls"), narListingData);
36+
} catch (SystemError &) {
37+
ignoreExceptionExceptInterrupt();
38+
}
39+
}
40+
41+
std::optional<std::string> NarCache::getNar(const Hash & narHash)
42+
{
43+
try {
44+
return nix::readFile(makeCacheFile(narHash, "nar"));
45+
} catch (SystemError &) {
46+
return std::nullopt;
47+
}
48+
}
49+
50+
std::string NarCache::getNarBytes(const Hash & narHash, uint64_t offset, uint64_t length)
51+
{
52+
auto cacheFile = makeCacheFile(narHash, "nar");
53+
54+
AutoCloseFD fd = toDescriptor(open(
55+
cacheFile.c_str(),
56+
O_RDONLY
57+
#ifndef _WIN32
58+
| O_CLOEXEC
59+
#endif
60+
));
61+
if (!fd)
62+
throw SysError("opening NAR cache file %s", cacheFile);
63+
64+
if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset)
65+
throw SysError("seeking in %s", cacheFile);
66+
67+
std::string buf(length, 0);
68+
readFull(fd.get(), buf.data(), length);
69+
return buf;
70+
}
71+
72+
std::optional<std::string> NarCache::getNarListing(const Hash & narHash)
73+
{
74+
try {
75+
return nix::readFile(makeCacheFile(narHash, "ls"));
76+
} catch (SystemError &) {
77+
return std::nullopt;
78+
}
79+
}
80+
81+
} // namespace nix

‎src/libstore/remote-fs-accessor.cc‎

Lines changed: 32 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,15 @@
11
#include <nlohmann/json.hpp>
22
#include "nix/store/remote-fs-accessor.hh"
33
#include "nix/store/nar-accessor.hh"
4-
5-
#include <sys/types.h>
6-
#include <sys/stat.h>
7-
#include <fcntl.h>
4+
#include "nix/store/nar-cache.hh"
85

96
namespace nix {
107

11-
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, std::filesystem::path cacheDir_)
8+
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, std::shared_ptr<NarCache> narCache)
129
: store(store)
1310
, requireValidPath(requireValidPath)
14-
, cacheDir(std::move(cacheDir_))
15-
{
16-
if (!cacheDir.empty())
17-
createDirs(cacheDir);
18-
}
19-
20-
std::filesystem::path RemoteFSAccessor::makeCacheFile(const Hash & narHash, const std::string & ext)
21-
{
22-
assert(!cacheDir.empty());
23-
return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
24-
}
25-
26-
ref<SourceAccessor> RemoteFSAccessor::addToCache(
27-
std::string_view hashPart,
28-
const std::filesystem::path & cacheFile,
29-
const std::filesystem::path & listingFile,
30-
std::string && nar)
11+
, narCache(std::move(narCache))
3112
{
32-
if (!cacheFile.empty()) {
33-
try {
34-
/* FIXME: do this asynchronously. */
35-
writeFile(cacheFile, nar);
36-
} catch (...) {
37-
ignoreExceptionExceptInterrupt();
38-
}
39-
}
40-
41-
auto narAccessor = makeNarAccessor(std::move(nar));
42-
nars.emplace(hashPart, narAccessor);
43-
44-
if (!listingFile.empty()) {
45-
try {
46-
nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
47-
writeFile(listingFile, j.dump());
48-
} catch (...) {
49-
ignoreExceptionExceptInterrupt();
50-
}
51-
}
52-
53-
return narAccessor;
5413
}
5514

5615
std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
@@ -67,55 +26,46 @@ std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath &
6726
if (i != nars.end())
6827
return i->second;
6928

70-
std::filesystem::path cacheFile, listingFile;
29+
Hash narHash{HashAlgorithm::SHA256};
7130

72-
if (!cacheDir.empty()) {
31+
if (narCache) {
7332
auto info = store->queryPathInfo(storePath);
33+
narHash = info->narHash;
7434

75-
cacheFile = makeCacheFile(info->narHash, "nar");
76-
listingFile = makeCacheFile(info->narHash, "ls");
77-
78-
if (nix::pathExists(cacheFile)) {
79-
try {
80-
auto listing = nix::readFile(listingFile);
81-
auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) {
82-
AutoCloseFD fd = toDescriptor(open(
83-
cacheFile.c_str(),
84-
O_RDONLY
85-
#ifndef _WIN32
86-
| O_CLOEXEC
87-
#endif
88-
));
89-
if (!fd)
90-
throw SysError("opening NAR cache file '%s'", cacheFile);
91-
92-
if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset)
93-
throw SysError("seeking in '%s'", cacheFile);
94-
95-
std::string buf(length, 0);
96-
readFull(fd.get(), buf.data(), length);
97-
98-
return buf;
35+
if (auto listingData = narCache->getNarListing(narHash)) {
36+
auto narAccessor =
37+
makeLazyNarAccessor(*listingData, [narCache(narCache), narHash](uint64_t offset, uint64_t length) {
38+
return narCache->getNarBytes(narHash, offset, length);
9939
});
10040

101-
nars.emplace(storePath.hashPart(), narAccessor);
102-
return narAccessor;
103-
104-
} catch (SystemError &) {
105-
}
41+
nars.emplace(storePath.hashPart(), narAccessor);
42+
return narAccessor;
43+
}
10644

107-
try {
108-
auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
109-
nars.emplace(storePath.hashPart(), narAccessor);
110-
return narAccessor;
111-
} catch (SystemError &) {
112-
}
45+
if (auto nar = narCache->getNar(narHash)) {
46+
auto narAccessor = makeNarAccessor(std::move(*nar));
47+
nars.emplace(storePath.hashPart(), narAccessor);
48+
return narAccessor;
11349
}
11450
}
11551

11652
StringSink sink;
11753
store->narFromPath(storePath, sink);
118-
return addToCache(storePath.hashPart(), cacheFile, listingFile, std::move(sink.s));
54+
55+
if (narCache) {
56+
StringSource source{sink.s};
57+
narCache->upsertNar(narHash, source);
58+
}
59+
60+
auto narAccessor = makeNarAccessor(std::move(sink.s));
61+
nars.emplace(storePath.hashPart(), narAccessor);
62+
63+
if (narCache) {
64+
nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
65+
narCache->upsertNarListing(narHash, j.dump());
66+
}
67+
68+
return narAccessor;
11969
}
12070

12171
std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)

0 commit comments

Comments
 (0)