Never throw VersionError (#1880)

* Allow downgrade + schema manip without version update

* It should work having two versions of the DB opened at the same time as long as they have a compatible schema.

A compatible schema does not need to have the same tables or indexes. They might differ quite a lot.
* Attaching an upgrader to one of them would mean that the upgrader runs even though the first one didn't request it. Therefore, this scenario should be avoided when having upgraders that modify content between versions.
* Schemas are not compatible if they index the same property or properties using different flags (unique or multiEntry)
* Schemas are not compatible if they have different primary keys

However, in most scenarios where we would fail opening one of the DBs, we will now succeed and be able to access the same data.

* Fixing a unit test

* Support infinite number of schema changes without version bump
(Not just 10)
When native verno is incremented 10 times to the next 10 multiple, create a table $meta that contains the virtual version. Maintain that table and delete it when no longer needed.

* Skip 2 tests in Dexie.Syncable's / Observable's integration tests

* Indentation only...

src/classes/dexie/dexie-open.ts

@@ -5,7 +5,7 @@ import { exceptions } from '../../errors';
 import { eventRejectHandler, preventDefault } from '../../functions/event-wrappers';
 import Promise, { wrap } from '../../helpers/promise';
 import { connections } from '../../globals/constants';
-import { runUpgraders, readGlobalSchema, adjustToExistingIndexNames, verifyInstalledSchema } from '../version/schema-helpers';
+import { runUpgraders, readGlobalSchema, adjustToExistingIndexNames, verifyInstalledSchema, patchCurrentVersion } from '../version/schema-helpers';
 import { safariMultiStoreFix } from '../../functions/quirks';
 import { _onDatabaseCreated } from '../../helpers/database-enumerator';
 import { vip } from './vip';
@@ -30,6 +30,8 @@ export function dexieOpen (db: Dexie) {
   state.dbOpenError = null;
   state.openComplete = false;
   const openCanceller = state.openCanceller;
+  let nativeVerToOpen = Math.round(db.verno * 10);
+  let schemaPatchMode = false;
 
   function throwIfCancelled() {
     // If state.openCanceller object reference is replaced, it means db.close() has been called,
@@ -44,19 +46,14 @@ export function dexieOpen (db: Dexie) {
       wasCreated = false;
 
   const tryOpenDB = () => new Promise((resolve, reject) => {
-    // Multiply db.verno with 10 will be needed to workaround upgrading bug in IE:
-    // IE fails when deleting objectStore after reading from it.
-    // A future version of Dexie.js will stopover an intermediate version to workaround this.
-    // At that point, we want to be backward compatible. Could have been multiplied with 2, but by using 10, it is easier to map the number to the real version number.
-
     throwIfCancelled();
     // If no API, throw!
     if (!indexedDB) throw new exceptions.MissingAPI();
     const dbName = db.name;
 
-    const req = state.autoSchema ?
+    const req = state.autoSchema || !nativeVerToOpen ?
       indexedDB.open(dbName) :
-      indexedDB.open(dbName, Math.round(db.verno * 10));
+      indexedDB.open(dbName, nativeVerToOpen);
     if (!req) throw new exceptions.MissingAPI(); // May happen in Safari private mode, see https://github.com/dfahlander/Dexie.js/issues/134
     req.onerror = eventRejectHandler(reject);
     req.onblocked = wrap(db._fireOnBlocked);
@@ -76,9 +73,12 @@ export function dexieOpen (db: Dexie) {
             });
         } else {
             upgradeTransaction.onerror = eventRejectHandler(reject);
-            var oldVer = e.oldVersion > Math.pow(2, 62) ? 0 : e.oldVersion; // Safari 8 fix.
+            const oldVer = e.oldVersion > Math.pow(2, 62) ? 0 : e.oldVersion; // Safari 8 fix.
             wasCreated = oldVer < 1;
             db.idbdb = req.result;
+            if (schemaPatchMode) {
+              patchCurrentVersion(db, upgradeTransaction);
+            }
             runUpgraders(db, oldVer / 10, upgradeTransaction, reject);
         }
     }, reject);
@@ -94,8 +94,12 @@ export function dexieOpen (db: Dexie) {
           if (state.autoSchema) readGlobalSchema(db, idbdb, tmpTrans);
           else {
               adjustToExistingIndexNames(db, db._dbSchema, tmpTrans);
-              if (!verifyInstalledSchema(db, tmpTrans)) {
-                  console.warn(`Dexie SchemaDiff: Schema was extended without increasing the number passed to db.version(). Some queries may fail.`);
+              if (!verifyInstalledSchema(db, tmpTrans) && !schemaPatchMode) {
+                console.warn(`Dexie SchemaDiff: Schema was extended without increasing the number passed to db.version(). Dexie will add missing parts and increment native version number to workaround this.`);
+                idbdb.close();
+                nativeVerToOpen = idbdb.version + 1;
+                schemaPatchMode = true;
+                return resolve (tryOpenDB()); // Try again with new version (nativeVerToOpen
               }
           }
           generateMiddlewareStacks(db, tmpTrans);
@@ -125,15 +129,24 @@ export function dexieOpen (db: Dexie) {
 
     }, reject);
   }).catch(err => {
-    if (err && err.name === 'UnknownError' && state.PR1398_maxLoop > 0) {
-      // Bug in Chrome after clearing site data
-      // https://github.com/dexie/Dexie.js/issues/543#issuecomment-1795736695
-      state.PR1398_maxLoop--;
-      console.warn('Dexie: Workaround for Chrome UnknownError on open()');
-      return tryOpenDB();
-    } else {
-      return Promise.reject(err);
+    switch (err?.name) {
+      case "UnknownError":
+        if (state.PR1398_maxLoop > 0) {
+          // Bug in Chrome after clearing site data
+          // https://github.com/dexie/Dexie.js/issues/543#issuecomment-1795736695
+          state.PR1398_maxLoop--;
+          console.warn('Dexie: Workaround for Chrome UnknownError on open()');
+          return tryOpenDB();
+        }
+        break;
+      case "VersionError":
+        if (nativeVerToOpen > 0) {
+          nativeVerToOpen = 0;
+          return tryOpenDB();
+        }
+        break;
     }
+    return Promise.reject(err);
   });
 
   // safari14Workaround = Workaround by jakearchibald for new nasty bug in safari 14.

src/classes/dexie/dexie.ts

@@ -59,6 +59,7 @@ export interface DbReadyState {
   autoSchema: boolean;
   vcFired?: boolean;
   PR1398_maxLoop?: number;
+  autoOpen?: boolean;
 }
 
 export class Dexie implements IDexie {
@@ -125,7 +126,8 @@ export class Dexie implements IDexie {
       cancelOpen: nop,
       openCanceller: null as Promise,
       autoSchema: true,
-      PR1398_maxLoop: 3
+      PR1398_maxLoop: 3,
+      autoOpen: options.autoOpen,
     };
     state.dbReadyPromise = new Promise(resolve => {
       state.dbReadyResolve = resolve;
@@ -268,7 +270,7 @@ export class Dexie implements IDexie {
         return reject(new exceptions.DatabaseClosed(this._state.dbOpenError));
       }
       if (!this._state.isBeingOpened) {
-        if (!this._options.autoOpen) {
+        if (!this._state.autoOpen) {
           reject(new exceptions.DatabaseClosed());
           return;
         }
@@ -325,9 +327,13 @@ export class Dexie implements IDexie {
   }
 
   close({disableAutoOpen} = {disableAutoOpen: true}): void {
+    const wasOpen = this.isOpen();
     this._close();
     const state = this._state;
-    if (disableAutoOpen) this._options.autoOpen = false;
+    if (disableAutoOpen) this._state.autoOpen = false;
+    else if (wasOpen && this._options.autoOpen) {
+      this._state.autoOpen = true;
+    }
     state.dbOpenError = new exceptions.DatabaseClosed();
     if (state.isBeingOpened)
       state.cancelOpen(state.dbOpenError);

src/classes/version/schema-helpers.ts

@@ -12,6 +12,8 @@ import { hasIEDeleteObjectStoreBug, isIEOrEdge } from '../../globals/constants';
 import { createIndexSpec, nameFromKeyPath } from '../../helpers/index-spec';
 import { createTableSchema } from '../../helpers/table-schema';
 import { generateMiddlewareStacks } from '../dexie/generate-middleware-stacks';
+import { debug } from '../../helpers/debug';
+import { PromiseExtended } from '../../public/types/promise-extended';
 
 export function setApiOnPlace(db: Dexie, objs: Object[], tableNames: string[], dbschema: DbSchema) {
   tableNames.forEach(tableName => {
@@ -54,6 +56,10 @@ export function lowerVersionFirst(a: Version, b: Version) {
 
 export function runUpgraders(db: Dexie, oldVersion: number, idbUpgradeTrans: IDBTransaction, reject) {
   const globalSchema = db._dbSchema;
+  if (idbUpgradeTrans.objectStoreNames.contains('$meta') && !globalSchema.$meta) {
+    globalSchema.$meta = createTableSchema("$meta", parseIndexSyntax("")[0], []);
+    db._storeNames.push('$meta');
+  }
   const trans = db._createTransaction('readwrite', db._storeNames, globalSchema);
   trans.create(idbUpgradeTrans);
   trans._completion.catch(reject);
@@ -69,14 +75,58 @@ export function runUpgraders(db: Dexie, oldVersion: number, idbUpgradeTrans: IDB
       });
       generateMiddlewareStacks(db, idbUpgradeTrans);
       Promise.follow(() => db.on.populate.fire(trans)).catch(rejectTransaction);
-    } else
-      updateTablesAndIndexes(db, oldVersion, trans, idbUpgradeTrans).catch(rejectTransaction);
+    } else {
+      generateMiddlewareStacks(db, idbUpgradeTrans);
+      return getExistingVersion(db, trans, oldVersion)
+        .then(oldVersion => updateTablesAndIndexes(db, oldVersion, trans, idbUpgradeTrans))
+        .catch(rejectTransaction);
+    }
   });
 }
 
 export type UpgradeQueueItem = (idbtrans: IDBTransaction) => PromiseLike<any> | void;
 
-export function updateTablesAndIndexes(
+export function patchCurrentVersion(db: Dexie, idbUpgradeTrans: IDBTransaction) {
+  createMissingTables(db._dbSchema, idbUpgradeTrans);
+  if (idbUpgradeTrans.db.version % 10 === 0 && !idbUpgradeTrans.objectStoreNames.contains('$meta')) {
+    // Rolled over to the next 10-ies due to many schema upgrades without bumping version.
+    // No problem! We pin the database to its expected version by adding the $meta table so that next
+    // time the programmer bumps the version and attaches, an upgrader, that upgrader will indeed run,
+    // as well any further upgraders coming after that.
+    idbUpgradeTrans.db.createObjectStore('$meta').add(Math.ceil((idbUpgradeTrans.db.version / 10) - 1), 'version');
+  }
+  const globalSchema = buildGlobalSchema(db, db.idbdb, idbUpgradeTrans);
+  adjustToExistingIndexNames(db, db._dbSchema, idbUpgradeTrans);
+  const diff = getSchemaDiff(globalSchema, db._dbSchema);
+  for (const tableChange of diff.change) {
+    if (tableChange.change.length || tableChange.recreate) {
+      console.warn(`Unable to patch indexes of table ${tableChange.name} because it has changes on the type of index or primary key.`);
+      return;
+    }
+    const store = idbUpgradeTrans.objectStore(tableChange.name);
+    tableChange.add.forEach(idx => {
+      if (debug) console.debug(`Dexie upgrade patch: Creating missing index ${tableChange.name}.${idx.src}`);
+      addIndex(store, idx);
+    });
+  }
+}
+
+function getExistingVersion(db: Dexie, trans: Transaction, oldVersion: number): PromiseExtended<number> {
+  // In normal case, existing version is the native installed version divided by 10.
+  // However, in case more than 10 schema changes have been made on the same version (such as while
+  // developing an app), the native version may have passed beyond a multiple of 10 within the same version.
+  // When that happens, a table $meta will have been created, containing a single entry with key "version"
+  // and the value of the real old version to use when running upgraders going forward.
+  if (trans.storeNames.includes('$meta')) {
+    return trans.table('$meta').get('version').then(metaVersion => {
+      return metaVersion != null ? metaVersion : oldVersion
+    })
+  } else {
+    return Promise.resolve(oldVersion);
+  }
+}
+
+function updateTablesAndIndexes(
   db: Dexie,
   oldVersion: number,
   trans: Transaction,
@@ -88,8 +138,21 @@ export function updateTablesAndIndexes(
   const versions = db._versions;
   let globalSchema = db._dbSchema = buildGlobalSchema(db, db.idbdb, idbUpgradeTrans);
   let anyContentUpgraderHasRun = false;
-
+  
   const versToRun = versions.filter(v => v._cfg.version >= oldVersion);
+  if (versToRun.length === 0) {
+    // Important not to continue at this point.
+    // Coming here means we've already patched schema in patchCurrentVersion() after having
+    // incremented native version to a value above the declared highest version.
+    // When being in this mode, it means that there might be different versions the db competing
+    // about it with different version of the schema. Therefore, we must avoid deleting tables
+    // or indexes here so that both versions can co-exist until the application has been upgraded to
+    // a version that declares no lower than the native version.
+    // If after that, a downgrade happens again, we'll end up here again, accepting both versions
+    // And we'll stay in this state until app developer releases a new declared version.
+    return Promise.resolve(); 
+  }
+
   versToRun.forEach(version => {
     queue.push(() => {
       const oldSchema = globalSchema;
@@ -178,6 +241,21 @@ export function updateTablesAndIndexes(
       setApiOnPlace(db, [db.Transaction.prototype], db._storeNames, db._dbSchema);
       trans.schema = db._dbSchema;
     });
+    // Maintain the $meta table after this version's tables and indexes has been created and content upgraders have run.
+    queue.push(idbtrans => {
+      if (db.idbdb.objectStoreNames.contains('$meta')) {
+        if (Math.ceil(db.idbdb.version / 10) === version._cfg.version) {
+          // Remove $meta table if it's no more needed - we are in line with the native version
+          db.idbdb.deleteObjectStore('$meta');
+          delete db._dbSchema.$meta;
+          db._storeNames = db._storeNames.filter(name => name !== '$meta');
+        } else {
+          // We're still not in line with the native version. Make sure to update the virtual version
+          // to the successfully run version
+          idbtrans.objectStore('$meta').put(version._cfg.version, 'version');
+        }
+      }
+    }); 
   });
 
   // Now, create a queue execution engine
@@ -285,6 +363,7 @@ export function createTable(
 export function createMissingTables(newSchema: DbSchema, idbtrans: IDBTransaction) {
   keys(newSchema).forEach(tableName => {
     if (!idbtrans.db.objectStoreNames.contains(tableName)) {
+      if (debug) console.debug('Dexie: Creating missing table', tableName);
       createTable(idbtrans, tableName, newSchema[tableName].primKey, newSchema[tableName].indexes);
     }
   });

src/functions/temp-transaction.ts

@@ -21,7 +21,7 @@ export function tempTransaction (
       return rejection(new exceptions.DatabaseClosed(db._state.dbOpenError));
     }
     if (!db._state.isBeingOpened) {
-      if (!db._options.autoOpen)
+      if (!db._state.autoOpen)
         return rejection(new exceptions.DatabaseClosed());
       db.open().catch(nop); // Open in background. If if fails, it will be catched by the final promise anyway.
     }

test/tests-misc.js

@@ -328,9 +328,13 @@ asyncTest("PR #1108", async ()=>{
     }
     const origConsoleWarn = console.warn;
     const warnings = [];
-    console.warn = function(msg){warnings.push(msg); return origConsoleWarn.apply(this, arguments)};
+    console.warn = function(msg){
+        warnings.push(msg);
+        return origConsoleWarn.apply(this, arguments)
+    };
     try {
         const DBNAME = "PR1108";
+        await Dexie.delete(DBNAME);
         let db = new Dexie(DBNAME);
         db.version(1).stores({
             foo: "id"