Merge pull request #525 from DeterminateSystems/nar-info-disk-cache-generic-fields

NarInfoDiskCache: Support arbitrary nix-cache-info fields

Author: edolstra

src/libstore-tests/nar-info-disk-cache.cc

@@ -15,6 +15,13 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
     int prio = 12345;
     bool wantMassQuery = true;
 
+    auto mkFields = [](bool wantMassQuery, int prio) {
+        return std::map<std::string, std::string>{
+            {"WantMassQuery", wantMassQuery ? "1" : "0"},
+            {"Priority", std::to_string(prio)},
+        };
+    };
+
     auto tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir);
     auto dbPath(tmpDir / "test-narinfo-disk-cache.sqlite");
@@ -30,22 +37,20 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
 
         // Set up "background noise" and check that different caches receive different ids
         {
-            auto bc1 =
-                cache->createCache("https://bar", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio});
-            auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.priority = 12});
+            auto bc1 = cache->createCache("https://bar", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)});
+            auto bc2 = cache->createCache("https://xyz", "/nix/storedir", {.fields = mkFields(false, 12)});
             ASSERT_NE(bc1, bc2);
             barId = bc1;
         }
 
         // Check that the fields are saved and returned correctly. This does not test
         // the select statement yet, because of in-memory caching.
-        savedId = cache->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio});
+        savedId = cache->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)});
         ;
         {
             auto r = cache->upToDateCacheExists("http://foo");
             ASSERT_TRUE(r);
-            ASSERT_EQ(r->priority, prio);
-            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+            ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio));
             ASSERT_EQ(savedId, r->id);
         }
 
@@ -68,8 +73,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
         {
             auto r = cache->upToDateCacheExists("http://foo");
             ASSERT_TRUE(r);
-            ASSERT_EQ(r->priority, prio);
-            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+            ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio));
         }
     }
 
@@ -85,13 +89,12 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
         }
 
         // "Update", same data, check that the id number is reused
-        cache2->createCache("http://foo", "/nix/storedir", {.wantMassQuery = wantMassQuery, .priority = prio});
+        cache2->createCache("http://foo", "/nix/storedir", {.fields = mkFields(wantMassQuery, prio)});
 
         {
             auto r = cache2->upToDateCacheExists("http://foo");
             ASSERT_TRUE(r);
-            ASSERT_EQ(r->priority, prio);
-            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+            ASSERT_EQ(r->fields, mkFields(wantMassQuery, prio));
             ASSERT_EQ(r->id, savedId);
         }
 
@@ -108,11 +111,9 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
             auto r0 = cache2->upToDateCacheExists("https://bar");
             ASSERT_FALSE(r0);
 
-            cache2->createCache(
-                "https://bar", "/nix/storedir", {.wantMassQuery = !wantMassQuery, .priority = prio + 10});
+            cache2->createCache("https://bar", "/nix/storedir", {.fields = mkFields(!wantMassQuery, prio + 10)});
             auto r = cache2->upToDateCacheExists("https://bar");
-            ASSERT_EQ(r->wantMassQuery, !wantMassQuery);
-            ASSERT_EQ(r->priority, prio + 10);
+            ASSERT_EQ(r->fields, mkFields(!wantMassQuery, prio + 10));
             ASSERT_EQ(r->id, barId);
         }
 

src/libstore/binary-cache-store.cc

@@ -45,8 +45,10 @@ BinaryCacheStore::BinaryCacheStore(Config & config)
     narMagic = sink.s;
 }
 
-void BinaryCacheStore::init()
+std::map<std::string, std::string> BinaryCacheStore::parseNixCacheInfo()
 {
+    std::map<std::string, std::string> fields;
+
     auto cacheInfo = getNixCacheInfo();
     if (!cacheInfo) {
         upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
@@ -64,13 +66,31 @@ void BinaryCacheStore::init()
                         config.getHumanReadableURI(),
                         value,
                         storeDir);
-            } else if (name == "WantMassQuery") {
-                config.wantMassQuery.setDefault(value == "1");
-            } else if (name == "Priority") {
-                config.priority.setDefault(std::stoi(value));
+            } else {
+                /* Keep every other field verbatim, including ones we
+                   don't (yet) understand. The known ones are applied
+                   by applyCacheInfoFields(). */
+                fields.insert_or_assign(name, value);
             }
         }
     }
+
+    return fields;
+}
+
+void BinaryCacheStore::applyCacheInfoFields(const std::map<std::string, std::string> & fields)
+{
+    if (auto * value = get(fields, "WantMassQuery"))
+        config.wantMassQuery.setDefault(*value == "1");
+    if (auto * value = get(fields, "Priority")) {
+        if (auto priority = string2Int<int>(*value))
+            config.priority.setDefault(*priority);
+    }
+}
+
+void BinaryCacheStore::init()
+{
+    applyCacheInfoFields(parseNixCacheInfo());
 }
 
 std::optional<std::string> BinaryCacheStore::getNixCacheInfo()

src/libstore/http-binary-cache-store.cc

@@ -69,16 +69,16 @@ void HttpBinaryCacheStore::init()
     auto cacheKey = config->getReference().render(/*withParams=*/false);
 
     if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) {
-        config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
-        config->priority.setDefault(cacheInfo->priority);
+        applyCacheInfoFields(cacheInfo->fields);
     } else {
+        std::map<std::string, std::string> fields;
         try {
-            BinaryCacheStore::init();
+            fields = parseNixCacheInfo();
         } catch (UploadToHTTP &) {
             throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string());
         }
-        diskCache->createCache(
-            cacheKey, config->storeDir, {.wantMassQuery = config->wantMassQuery, .priority = config->priority});
+        applyCacheInfoFields(fields);
+        diskCache->createCache(cacheKey, config->storeDir, {.fields = std::move(fields)});
     }
 }
 

src/libstore/include/nix/store/binary-cache-store.hh

@@ -98,6 +98,16 @@ protected:
 
     BinaryCacheStore(Config &);
 
+    /**
+     * Fetch and parse `nix-cache-info`.
+     */
+    std::map<std::string, std::string> parseNixCacheInfo();
+
+    /**
+     * Apply the known `nix-cache-info` fields from `fields` to this store.
+     */
+    void applyCacheInfoFields(const std::map<std::string, std::string> & fields);
+
     /**
      * Compute the path to the given realisation
      *

src/libstore/include/nix/store/nar-info-disk-cache.hh

@@ -5,6 +5,9 @@
 #include "nix/store/nar-info.hh"
 #include "nix/store/realisation.hh"
 
+#include <map>
+#include <string>
+
 namespace nix {
 
 struct SQLiteSettings;
@@ -28,8 +31,13 @@ struct NarInfoDiskCache
     struct CacheInfo
     {
         int id = 0;
-        bool wantMassQuery = false;
-        int priority = 0;
+
+        /**
+         * The `nix-cache-info` fields other than `StoreDir`, stored
+         * verbatim (e.g. `WantMassQuery`).  Keeping these generic
+         * means fields we don't (yet) understand are still recorded.
+         */
+        std::map<std::string, std::string> fields;
     };
 
     /**

src/libstore/nar-info-disk-cache.cc

@@ -19,8 +19,7 @@ create table if not exists BinaryCaches (
     url       text unique not null,
     timestamp integer not null,
     storeDir  text not null,
-    wantMassQuery integer not null,
-    priority  integer not null
+    fields    text not null
 );
 
 create table if not exists NARs (
@@ -84,7 +83,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
     NarInfoDiskCacheImpl(
         const Settings & settings,
         SQLiteSettings sqliteSettings,
-        std::filesystem::path dbPath = getCacheDir() / "binary-cache-detsys-v1.sqlite")
+        std::filesystem::path dbPath = getCacheDir() / "binary-cache-detsys-v2.sqlite")
         : NarInfoDiskCache{settings}
     {
         auto state(_state.lock());
@@ -99,11 +98,10 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
 
         state->insertCache.create(
             state->db,
-            "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;");
+            "insert into BinaryCaches(url, timestamp, storeDir, fields) values (?1, ?2, ?3, ?4) on conflict (url) do update set timestamp = ?2, storeDir = ?3, fields = ?4 returning id;");
 
         state->queryCache.create(
-            state->db,
-            "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?");
+            state->db, "select id, storeDir, fields from BinaryCaches where url = ? and timestamp > ?");
 
         state->insertNAR.create(
             state->db,
@@ -193,8 +191,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
                 .storeDir = queryCache.getStr(1),
                 .info = {
                     .id = (int) queryCache.getInt(0),
-                    .wantMassQuery = queryCache.getInt(2) != 0,
-                    .priority = (int) queryCache.getInt(3),
+                    .fields = nlohmann::json::parse(queryCache.getStr(2)).get<std::map<std::string, std::string>>(),
                 }};
             state.caches.emplace(uri, cache);
         }
@@ -222,8 +219,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
                            .apply(uri)
                            .apply(time(nullptr))
                            .apply(storeDir)
-                           .apply(info.wantMassQuery)
-                           .apply(info.priority));
+                           .apply(nlohmann::json(info.fields).dump()));
                 if (!r.next()) {
                     unreachable();
                 }