feat: import config metadata and import to projectSettings (#721)

* * use new tables for `projectSettings`
* import metadata from config
* start solving tests

* add metadata to every config make loading of config files sequential

* add metadata specific unit tests

* fix a type error

* add mapeo-mock-data with mapeo-schema updated

* re-generate drizzle, add `configMetadata` to `configImport`

* update metadata.json on fixtures, fix wrong type on validation

* import date should not exist on config file, but added on import

* add local version of default-config

* add parsing and validation of `fileVersion` which is a COMVER string now

* revert use of iterpal

* missed conflict on drizzle journal and snapshot

* another missed conflict...

* remove locally installed versions of dependencies

* config-import: `importDate` should be defined when calling import, not when calling `get metadata`

* add updated config fixtures, update @mapeo/default config, fix config.fileVersion parsing

* remove unnecessary @ts-ignore

* add metadata to config fixtures

* missing commitiing metadata.json to validConfig

* add metadata.json to completeConfig

* delete all zips (don't know how they got here...)

* recover missing metadata fixtures, remove unnecessary tests

---------

Co-authored-by: Tomás Ciccola <tciccola@digital-democracy.com>

Author: tomasciccola

drizzle/client/0000_organic_bloodstrike.sql

@@ -30,4 +30,4 @@ CREATE TABLE `projectSettings` (
 	`forks` text NOT NULL
 );
 --> statement-breakpoint
-CREATE UNIQUE INDEX `localDeviceInfo_deviceId_unique` ON `localDeviceInfo` (`deviceId`);
+CREATE UNIQUE INDEX `localDeviceInfo_deviceId_unique` ON `localDeviceInfo` (`deviceId`);

drizzle/client/meta/0000_snapshot.json

@@ -196,4 +196,4 @@
     "tables": {},
     "columns": {}
   }
-}
+}

drizzle/client/meta/_journal.json

@@ -10,4 +10,4 @@
       "breakpoints": true
     }
   ]
-}
+}

drizzle/project/meta/0000_snapshot.json

@@ -1141,4 +1141,4 @@
     "tables": {},
     "columns": {}
   }
-}
+}

drizzle/project/meta/_journal.json

@@ -10,4 +10,4 @@
       "breakpoints": true
     }
   ]
-}
+}

package-lock.json

@@ -107,7 +107,7 @@
   "homepage": "https://github.com/digidem/mapeo-core#readme",
   "devDependencies": {
     "@bufbuild/buf": "^1.26.1",
-    "@mapeo/default-config": "4.0.0-alpha.5",
+    "@mapeo/default-config": "4.0.0-alpha.6",
     "@mapeo/mock-data": "^1.0.3-alpha.1",
     "@sinonjs/fake-timers": "^10.0.2",
     "@types/b4a": "^1.6.0",

package.json

@@ -4,6 +4,7 @@ import { json, buffer } from 'node:stream/consumers'
 import { assert } from './utils.js'
 import path from 'node:path'
 import { parse as parseBCP47 } from 'bcp-47'
+import { SUPPORTED_CONFIG_VERSION } from './constants.js'
 
 // Throw error if a zipfile contains more than 10,000 entries
 const MAX_ENTRIES = 10_000
@@ -26,6 +27,8 @@ const MAX_ICON_SIZE = 10_000_000
  * }} TranslationsFile
  */
 
+/** @typedef {NonNullable<import('@mapeo/schema').ProjectSettingsValue['configMetadata']>} MetadataFile */
+
 /**
  * @typedef {Parameters<import('./icon-api.js').IconApi['create']>[0]} IconData
  */
@@ -36,23 +39,31 @@ const MAX_ICON_SIZE = 10_000_000
 export async function readConfig(configPath) {
   /** @type {Error[]} */
   const warnings = []
+  const importDate = new Date().toISOString()
 
   const zip = await yauzl.open(configPath)
   if (zip.entryCount > MAX_ENTRIES) {
     // MAX_ENTRIES in MAC can be inacurrate
     throw new Error(`Zip file contains too many entries. Max is ${MAX_ENTRIES}`)
   }
   const entries = await zip.readEntries(MAX_ENTRIES)
-  const [presetsFile, translationsFile] = await Promise.all([
-    findPresetsFile(entries),
-    findTranslationsFile(entries),
-  ])
+  const presetsFile = await findPresetsFile(entries)
+  const translationsFile = await findTranslationsFile(entries)
+  const metadataFile = await findMetadataFile(entries)
+  assert(
+    isValidConfigFile(metadataFile),
+    `invalid or missing config file version ${metadataFile.fileVersion}. We support version ${SUPPORTED_CONFIG_VERSION}}`
+  )
 
   return {
     get warnings() {
       return warnings
     },
 
+    get metadata() {
+      return { ...metadataFile, importDate }
+    },
+
     async close() {
       zip.close()
     },
@@ -266,6 +277,27 @@ async function findTranslationsFile(entries) {
   return result
 }
 
+/**
+ * @param {ReadonlyArray<Entry>} entries
+ * @returns {Promise<Omit<MetadataFile, 'importDate'>>}
+ */
+async function findMetadataFile(entries) {
+  const metadataEntry = entries.find(
+    (entry) => entry.filename === 'metadata.json'
+  )
+  assert(metadataEntry, 'Zip file does not contain metadata.json')
+  let result
+  try {
+    result = await json(await metadataEntry.openReadStream())
+  } catch (err) {
+    throw new Error('Could not parse metadata.json')
+  }
+  assert(isRecord(result), 'Invalid metadata.json file')
+  assert(isValidMetadataFile(result), 'Invalid structure of metadata file')
+
+  return result
+}
+
 /**
  * @param {Error[]} warnings
  */
@@ -487,6 +519,22 @@ function parseIcon(filename, buf) {
   }
 }
 
+/**
+ * @param {Record<string,unknown>} obj
+ * @returns {obj is Omit<MetadataFile, 'importDate'>}
+ */
+function isValidMetadataFile(obj) {
+  // extra fields are valid
+  return (
+    'name' in obj &&
+    'buildDate' in obj &&
+    'fileVersion' in obj &&
+    typeof obj['name'] === 'string' &&
+    typeof obj['buildDate'] === 'string' &&
+    typeof obj['fileVersion'] === 'string'
+  )
+}
+
 /**
  * @param {Record<string, unknown>} message
  * @returns {message is Record<string,{label:string, value:string}>}
@@ -512,3 +560,21 @@ function isRecord(value) {
 function hasOwn(obj, prop) {
   return Object.prototype.hasOwnProperty.call(obj, prop)
 }
+
+/**
+ * @param {Object} obj
+ * @param {string | undefined} [obj.fileVersion]
+ * @returns {boolean}
+ */
+function isValidConfigFile({ fileVersion }) {
+  if (!fileVersion) return false
+  const regex = /^(\d+)\.(\d+)$/
+  const match = fileVersion.match(regex)
+
+  if (!match) return false
+
+  const major = parseInt(match[1], 10)
+  //const minor = parseInt(match[2], 10)
+
+  return major >= SUPPORTED_CONFIG_VERSION
+}

src/config-import.js

@@ -20,3 +20,5 @@ export const NAMESPACE_SCHEMAS = /** @type {const} */ ({
   ],
   auth: ['coreOwnership', 'role'],
 })
+
+export const SUPPORTED_CONFIG_VERSION = 1

src/constants.js

@@ -51,6 +51,7 @@ import { readConfig } from './config-import.js'
 import TranslationApi from './translation-api.js'
 
 /** @typedef {Omit<import('@mapeo/schema').ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
+/** @typedef {import('@mapeo/schema').ProjectSettingsValue['configMetadata']} ConfigMetadata */
 
 const CORESTORE_STORAGE_FOLDER_NAME = 'corestore'
 const INDEXER_STORAGE_FOLDER_NAME = 'indexer'
@@ -838,6 +839,7 @@ export class MapeoProject extends TypedEmitter {
           vertex: [],
           relation: [],
         },
+        configMetadata: config.metadata,
       })
       this.#loadingConfig = false
       return config.warnings