Enable batched upgrades in SQL providers. Aim to pull 1MB of data out… (#59)

* Enable batched upgrades in SQL providers. Aim to pull 1MB of data out of the DB at a time to avoid potential OOM conditions when upgrading large databases
Allow consumers of NoSqlProvider to provide an estimated size for their database objects. This will help do better batching without having to do an initial transaction to see what object sizes look like

Author: berickson1

src/NoSqlProvider.ts

@@ -35,6 +35,8 @@ export interface StoreSchema {
     name: string;
     indexes?: IndexSchema[];
     primaryKeyPath: KeyPathType;
+    // Estimated object size to enable batched data migration. Default = 200
+    estimatedObjBytes?: number;
 }
 
 // Schema representing a whole database (a collection of stores).  Change your version number whenever you change your schema or

src/SqlProviderBase.ts

@@ -43,6 +43,9 @@ const schemaVersionKey = 'schemaVersion';
 // This was taked from the sqlite documentation
 const SQLITE_MAX_SQL_LENGTH_IN_BYTES = 1000000;
 
+const DB_SIZE_ESIMATE_DEFAULT = 200;
+const DB_MIGRATION_MAX_BYTE_TARGET = 1000000;
+
 interface IndexMetadata {
     key: string;
     storeName: string;
@@ -434,11 +437,20 @@ export abstract class SqlProviderBase extends NoSqlProvider.DbProvider {
                                 // Migrate the data over using our existing put functions
                                 // (since it will do the right things with the indexes)
                                 // and delete the temp table.
-                                const jsMigrator = () => {
+                                const jsMigrator = (batchOffset = 0): SyncTasks.Promise<any> => {
+                                    let esimatedSize = storeSchema.estimatedObjBytes || DB_SIZE_ESIMATE_DEFAULT;
+                                    let batchSize = Math.max(1, Math.floor(DB_MIGRATION_MAX_BYTE_TARGET / esimatedSize));
                                     let store = trans.getStore(storeSchema.name);
-                                    return trans.internal_getResultsFromQuery('SELECT nsp_data FROM temp_' + storeSchema.name)
-                                    .then(objs => {
-                                        return store.put(objs);
+                                    return trans.internal_getResultsFromQuery('SELECT nsp_data FROM temp_' + storeSchema.name + ' LIMIT ' +
+                                            batchSize + ' OFFSET ' + batchOffset)
+                                        .then(objs => {
+                                            return store.put(objs).then(() => {
+                                                // Are we done migrating?
+                                                if (objs.length < batchSize) {
+                                                    return undefined;
+                                                }
+                                                return jsMigrator(batchOffset + batchSize);
+                                            });
                                     });
                                 };
 
@@ -449,7 +461,9 @@ export abstract class SqlProviderBase extends NoSqlProvider.DbProvider {
                                     ])
                                     .then(createTempTable)
                                     .then(tableMaker)
-                                    .then(jsMigrator)
+                                    .then(() => {
+                                        return jsMigrator();
+                                    })
                                     .then(dropTempTable)
                                 );
                             } 

src/tests/NoSqlProviderTests.ts

@@ -12,7 +12,6 @@ import { IndexedDbProvider } from '../IndexedDbProvider';
 import { WebSqlProvider } from '../WebSqlProvider';
 
 import NoSqlProviderUtils = require('../NoSqlProviderUtils');
-
 import { SqlTransaction } from '../SqlProviderBase';
 
 // Don't trap exceptions so we immediately see them with a stack trace
@@ -1200,6 +1199,76 @@ describe('NoSqlProvider', function () {
                         });
                     });
 
+                    function testBatchUpgrade(expectedCallCount: number, itemByteSize: number): SyncTasks.Promise<void> {
+                        const recordCount = 5000;
+                        const data: { [id: string]: { id: string, tt: string } } = {};
+                        _.times(recordCount, num => {
+                            data[num.toString()] = {
+                                id: num.toString(),
+                                tt: 'tt' + num.toString()
+                            };
+                        });
+                        return openProvider(provName, {
+                            version: 1,
+                            stores: [
+                                {
+                                    name: 'test',
+                                    primaryKeyPath: 'id',
+                                    estimatedObjBytes: itemByteSize
+                                }
+                            ]
+                        }, true).then(prov => {
+                            return prov.put('test', _.values(data)).then(() => {
+                                return prov.close();
+                            });
+                        }).then(() => {
+                            let transactionSpy: sinon.SinonSpy | undefined;
+                            if (provName.indexOf('sql') !== -1) {
+                                // Check that we batch the upgrade by spying on number of queries indirectly
+                                // This only affects sql-based tests
+                                transactionSpy = sinon.spy(SqlTransaction.prototype, 'internal_getResultsFromQuery');
+                            }
+                            return openProvider(provName, {
+                                version: 2,
+                                stores: [
+                                    {
+                                        name: 'test',
+                                        primaryKeyPath: 'id',
+                                        estimatedObjBytes: itemByteSize,
+                                        indexes: [{
+                                            name: 'ind1',
+                                            keyPath: 'tt'
+                                        }]
+                                    }
+                                ]
+                            }, false).then(prov => {
+                                if (transactionSpy) {
+                                    assert.equal(transactionSpy.callCount, expectedCallCount, 'Incorrect transaction count');
+                                    transactionSpy.restore();
+                                }
+                                return prov.getAll('test', undefined).then((records: any) => {
+                                    assert.equal(records.length, _.keys(data).length, 'Incorrect record count');
+                                    _.each(records, dbRecordToValidate => {
+                                        const originalRecord = data[dbRecordToValidate.id];
+                                        assert.ok(!!originalRecord);
+                                        assert.equal(originalRecord.id, dbRecordToValidate.id);
+                                        assert.equal(originalRecord.tt, dbRecordToValidate.tt);
+                                    });
+                                }).then(() => {
+                                    return prov.close();
+                                });
+                            });
+                        });
+                    }
+
+                    it('Add index - Large records - batched upgrade', () => {
+                        return testBatchUpgrade(51, 10000);
+                    });
+
+                    it('Add index - small records - No batch upgrade', () => {
+                        return testBatchUpgrade(1, 1);
+                    });
+
                     if (provName.indexOf('indexeddb') !== 0) {
                         // This migration works on indexeddb because we don't check the types and the browsers silently accept it but just
                         // neglect to index the field...