WIP chores: archive < 2026 migrations

lib/model/knex-migrator.js

@@ -26,7 +26,7 @@ const withKnex = (config) => (mutator) => {
 };
 
 // Given a database, initiates migrations on it.
-const migrate = (db) => db.migrate.latest({ directory: `${__dirname}/migrations` });
+const migrate = (db) => db.migrate.latest({ directory: [`${__dirname}/migrations/archive`, `${__dirname}/migrations`] });
 
 module.exports = { knexConnect, withKnex, migrate };
 

lib/model/knexfile.js

@@ -35,5 +35,11 @@ setlibpqEnv(config.get('default.database'));
 module.exports = {
   client: 'pg',
   connection: {},
+  migrations: {
+    directory: [
+      './migrations/archive',
+      './migrations'
+    ]
+  }
 };
 

lib/model/migrations/archive/.eslintrc.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../../../.eslintrc.json",
+  "rules": {
+    "no-restricted-modules": [ "error", { "patterns": [ "../../*", "!../../pure-sql-migration" ] } ]
+  }
+}

lib/model/migrations/20180727-02-add-md5-to-blobs.js → lib/model/migrations/archive/20180727-02-add-md5-to-blobs.js

@@ -8,7 +8,7 @@
 // except according to the terms contained in the LICENSE file.
 //
 
-const { md5sum } = require('../../util/crypto'); // eslint-disable-line no-restricted-modules
+const { md5sum } = require('../../../util/crypto'); // eslint-disable-line no-restricted-modules
 
 const up = (knex) =>
   knex.schema.table('blobs', (blobs) => { blobs.string('md5', 32); })

lib/model/migrations/20180727-03-add-form-attachments-table.js → lib/model/migrations/archive/20180727-03-add-form-attachments-table.js

@@ -23,7 +23,7 @@ const up = (knex) =>
 
     fa.index([ 'formId' ]);
   }).then(() => {
-    const { expectedFormAttachments } = require('../../data/schema'); // eslint-disable-line no-restricted-modules
+    const { expectedFormAttachments } = require('../../../data/schema'); // eslint-disable-line no-restricted-modules
     const { uniq, pluck } = require('ramda');
 
     // now add all expected attachments on extant forms.

lib/model/migrations/20190520-01-add-form-versioning.js → lib/model/migrations/archive/20190520-01-add-form-versioning.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { shasum, sha256sum } = require('../../util/crypto'); // eslint-disable-line no-restricted-modules
+const { shasum, sha256sum } = require('../../../util/crypto'); // eslint-disable-line no-restricted-modules
 const assert = require('assert').strict;
 
 const check = (message, query) =>

lib/model/migrations/20191007-01-backfill-client-audits.js → lib/model/migrations/archive/20191007-01-backfill-client-audits.js

@@ -7,9 +7,9 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { parseClientAudits } = require('../../data/client-audits'); // eslint-disable-line no-restricted-modules
-const { getFormFields } = require('../../data/schema'); // eslint-disable-line no-restricted-modules
-const { traverseXml, findOne, root, node, text } = require('../../util/xml'); // eslint-disable-line no-restricted-modules
+const { parseClientAudits } = require('../../../data/client-audits'); // eslint-disable-line no-restricted-modules
+const { getFormFields } = require('../../../data/schema'); // eslint-disable-line no-restricted-modules
+const { traverseXml, findOne, root, node, text } = require('../../../util/xml'); // eslint-disable-line no-restricted-modules
 
 const up = (db) => new Promise((resolve, reject) => {
   const work = [];

lib/model/migrations/20191231-02-add-schema-storage.js → lib/model/migrations/archive/20191231-02-add-schema-storage.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { getFormFields } = require('../../data/schema'); // eslint-disable-line no-restricted-modules
+const { getFormFields } = require('../../../data/schema'); // eslint-disable-line no-restricted-modules
 
 const up = async (db) => {
   await db.schema.createTable('form_fields', (fields) => {
@@ -51,7 +51,7 @@ const up = async (db) => {
               // this config hardcoding would be dangerous with tests except that
               // tests will never invoke this path.
               const config = require('config').get('default.database');
-              const db2 = require('../knex-migrator').knexConnect(config); // eslint-disable-line no-restricted-modules
+              const db2 = require('../../knex-migrator').knexConnect(config); // eslint-disable-line no-restricted-modules
               return db2.select('projectId', 'xmlFormId').from('forms').where({ currentDefId: formDefId })
                 .then(([{ projectId, xmlFormId }]) => {
                   process.stderr.write(`\n!!!!\nThe database upgrade to v0.8 has failed because the Form '${xmlFormId}' in Project ${projectId} has an invalid schema. It tries to bind multiple instance nodes at the path ${path}.\n!!!!\n\n`);

lib/model/migrations/20200220-01-repair-submission-parsing.js → lib/model/migrations/archive/20200220-01-repair-submission-parsing.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { Submission } = require('../frames'); // eslint-disable-line no-restricted-modules
+const { Submission } = require('../../frames'); // eslint-disable-line no-restricted-modules
 
 const up = async (db) => {
   const work = [];

lib/model/migrations/20210120-01-instance-names.js → lib/model/migrations/archive/20210120-01-instance-names.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { Submission } = require('../frames'); // eslint-disable-line no-restricted-modules
+const { Submission } = require('../../frames'); // eslint-disable-line no-restricted-modules
 
 const up = async (db) => {
   await db.schema.table('submission_defs', (sds) => {

lib/model/migrations/20211008-01-track-select-many-options.js → lib/model/migrations/archive/20211008-01-track-select-many-options.js

@@ -8,10 +8,10 @@
 // except according to the terms contained in the LICENSE file.
 
 const { map } = require('ramda');
-const { getFormFields } = require('../../data/schema'); // eslint-disable-line no-restricted-modules
-const { getSelectMultipleResponses } = require('../../data/submission'); // eslint-disable-line no-restricted-modules
-const { Form } = require('../frames'); // eslint-disable-line no-restricted-modules
-const { construct } = require('../../util/util'); // eslint-disable-line no-restricted-modules
+const { getFormFields } = require('../../../data/schema'); // eslint-disable-line no-restricted-modules
+const { getSelectMultipleResponses } = require('../../../data/submission'); // eslint-disable-line no-restricted-modules
+const { Form } = require('../../frames'); // eslint-disable-line no-restricted-modules
+const { construct } = require('../../../util/util'); // eslint-disable-line no-restricted-modules
 
 const up = async (db) => {
   // add select many flag, options field to fields

lib/model/migrations/20230109-01-add-form-schema.js → lib/model/migrations/archive/20230109-01-add-form-schema.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { getFormFields, compare } = require('../../data/schema'); // eslint-disable-line no-restricted-modules
+const { getFormFields, compare } = require('../../../data/schema'); // eslint-disable-line no-restricted-modules
 
 /* Steps of this migration
   1. remove check field collision trigger

lib/model/migrations/20250927-01-geoextracts.js → lib/model/migrations/archive/20250927-01-geoextracts.js

@@ -7,4 +7,4 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-module.exports = require('../pure-sql-migration')(__filename);
+module.exports = require('../../pure-sql-migration')(__filename);

lib/model/migrations/20250928-01-backfill-submission-geocache-createfunction.js → lib/model/migrations/archive/20250928-01-backfill-submission-geocache-createfunction.js

@@ -7,4 +7,4 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-module.exports = require('../pure-sql-migration')(__filename);
+module.exports = require('../../pure-sql-migration')(__filename);

lib/model/migrations/20251127-01-submission-event-stamping.js → lib/model/migrations/archive/20251127-01-submission-event-stamping.js

@@ -7,4 +7,4 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-module.exports = require('../pure-sql-migration')(__filename);
+module.exports = require('../../pure-sql-migration')(__filename);

lib/model/migrations/20251201-01-entities-uuid-column-uuid-datatype.js → lib/model/migrations/archive/20251201-01-entities-uuid-column-uuid-datatype.js

@@ -7,4 +7,4 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-module.exports = require('../pure-sql-migration')(__filename);
+module.exports = require('../../pure-sql-migration')(__filename);

lib/model/migrations/20251209-01-ignore-leading-zeroes-for-geojson-geocomponents.js → lib/model/migrations/archive/20251209-01-ignore-leading-zeroes-for-geojson-geocomponents.js

@@ -7,4 +7,4 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-module.exports = require('../pure-sql-migration')(__filename);
+module.exports = require('../../pure-sql-migration')(__filename);

test/db-migrations/migrator.js

@@ -18,7 +18,7 @@ const { execSync } = require('node:child_process');
 const migrationsDir = './lib/model/migrations';
 const holdingPen = './test/db-migrations/.holding-pen';
 
-fs.mkdirSync(holdingPen, { recursive: true });
+fs.mkdirSync(`${holdingPen}/archive`, { recursive: true });
 
 restoreMigrations(); // eslint-disable-line no-use-before-define
 const allMigrations = loadMigrationsList(); // eslint-disable-line no-use-before-define
@@ -52,7 +52,7 @@ function runIncluding(lastMigrationToRun) {
 }
 
 function getIndex(migrationName) {
-  const idx = allMigrations.indexOf(migrationName);
+  const idx = allMigrations.findIndex(f => f.endsWith(migrationName));
   log('getIndex()', migrationName, 'found at', idx);
   if (idx === -1) throw new Error(`Unknown migration: ${migrationName}`);
   return idx;
@@ -67,16 +67,16 @@ function moveMigrationsToHoldingPen() {
 }
 
 function moveAll(src, tgt) {
-  fs.readdirSync(src)
+  fs.readdirSync(src, { recursive: true })
     .filter(f => f.endsWith('.js'))
     .forEach(f => fs.renameSync(`${src}/${f}`, `${tgt}/${f}`));
 }
 
 function loadMigrationsList() {
-  const migrations = fs.readdirSync(migrationsDir)
+  const migrations = fs.readdirSync(migrationsDir, { recursive: true })
     .filter(f => f.endsWith('.js'))
     .map(f => f.replace(/\.js$/, ''))
-    .sort(); // match sorting in pg-migrator and knex's fs-migrations.js
+    .sort((a, b) => a.replace(/^.*\//, '').localeCompare(b.replace(/^.*\//, ''))); // match sorting in pg-migrator and knex's fs-migrations.js (by default: collect all files in all directories)
   log();
   log('All migrations:');
   log();

test/integration/other/knex-migrations.js

@@ -20,7 +20,7 @@ const populateUsers = require('../fixtures/01-users');
 const { getFormFields } = require('../../../lib/data/schema');
 
 const withTestDatabase = withKnex(config.get('test.database'));
-const migrationsDir = appRoot + '/lib/model/migrations';
+const migrationsDir = [appRoot + '/lib/model/migrations/archive', appRoot + '/lib/model/migrations'];
 const upToMigration = (toName, inclusive = true) => withTestDatabase(async (migrator) => {
   await migrator.raw('drop owned by current_user');
   const migrations = await migrator.migrate.list({ directory: migrationsDir });

test/integration/setup.js

@@ -82,7 +82,7 @@ const initialize = async () => {
     await migrator.raw('drop owned by current_user');
     // Silence logging from migrations.
     console.log = noop; // eslint-disable-line no-console
-    await migrator.migrate.latest({ directory: appRoot + '/lib/model/migrations' });
+    await migrator.migrate.latest({ directory: [appRoot + '/lib/model/migrations/archive', appRoot + '/lib/model/migrations'] });
   } finally {
     console.log = log; // eslint-disable-line no-console
     await migrator.destroy();