feat: migration 10 to allow upgrading level in the browser

We use the [level](https://www.npmjs.com/package/level) module to supply
either [leveldown](http://npmjs.com/package/leveldown) or
[level-js](https://www.npmjs.com/package/level-js)
to [datastore-level](https://www.npmjs.com/package/datastore-level) depending
on if we're running under node or in the browser.

`[email protected]` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`
which includes the changes from
[Level/level-js#179](Level/level-js#179)
so `>5.x.x` requires all database keys/values to be Uint8Arrays and they
can no longer be strings.

We already store values as Uint8Arrays but our keys are strings, so here
we add a migration to converts all datastore keys to Uint8Arrays.

N.b. `leveldown` already does this conversion for us so this migration
only needs to run in the browser.

README.md

@@ -42,6 +42,11 @@ This package is inspired by the [go-ipfs repo migration tool](https://github.com
   - [Tests](#tests)
   - [Empty migrations](#empty-migrations)
   - [Migrations matrix](#migrations-matrix)
+  - [Migrations](#migrations)
+    - [7](#7)
+    - [8](#8)
+    - [9](#9)
+    - [10](#10)
 - [Developer](#developer)
   - [Module versioning notes](#module-versioning-notes)
 - [Contribute](#contribute)
@@ -268,6 +273,24 @@ This will create an empty migration with the next version.
 |                  8 | v0.48.0          |
 |                  9 | v0.49.0          |
 
+### Migrations
+
+#### 7
+
+This is the initial version of the datastore, inherited from go-IPFS in an attempt to maintain cross-compatibility between the two implementations.
+
+#### 8
+
+Blockstore keys are transformed into base32 representations of the multihash from the CID of the block.
+
+#### 9
+
+Pins were migrated from a DAG to a Datastore - see [ipfs/js-ipfs#2771](https://github.com/ipfs/js-ipfs/pull/2771)
+
+#### 10
+
+`[email protected]` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`.  This update requires a database migration to convert all string keys/values into buffers. Only runs in the browser, node is unaffected. See [Level/level-js#179](https://github.com/Level/level-js/pull/179)
+
 ## Developer
 
 ### Module versioning notes

migrations/index.js

@@ -16,5 +16,6 @@ module.exports = [
   Object.assign({version: 6}, emptyMigration),
   Object.assign({version: 7}, emptyMigration),
   require('./migration-8'),
-  require('./migration-9')
+  require('./migration-9'),
+  require('./migration-10')
 ]

migrations/migration-10/index.js

@@ -0,0 +1,157 @@
+'use strict'
+
+const {
+  createStore,
+  findLevelJs
+} = require('../../src/utils')
+const { Key } = require('interface-datastore')
+const fromString = require('uint8arrays/from-string')
+const toString = require('uint8arrays/to-string')
+
+async function keysToBinary (name, store, onProgress = () => {}) {
+  let db = findLevelJs(store)
+
+  // only interested in level-js
+  if (!db) {
+    onProgress(`${name} did not need an upgrade`)
+
+    return
+  }
+
+  onProgress(`Upgrading ${name}`)
+
+  await withEach(db, (key, value) => {
+    return [
+      { type: 'del', key: key },
+      { type: 'put', key: fromString(key), value: value }
+    ]
+  })
+}
+
+async function keysToStrings (name, store, onProgress = () => {}) {
+  let db = findLevelJs(store)
+
+  // only interested in level-js
+  if (!db) {
+    onProgress(`${name} did not need a downgrade`)
+
+    return
+  }
+
+  onProgress(`Downgrading ${name}`)
+
+  await withEach(db, (key, value) => {
+    return [
+      { type: 'del', key: key },
+      { type: 'put', key: toString(key), value: value }
+    ]
+  })
+}
+
+async function process (repoPath, repoOptions, onProgress, fn) {
+  const datastores = Object.keys(repoOptions.storageBackends)
+    .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore')
+    .map(name => ({
+      name,
+      store: createStore(repoPath, name, repoOptions)
+    }))
+
+  onProgress(0, `Migrating ${datastores.length} dbs`)
+  let migrated = 0
+
+  for (const { name, store } of datastores) {
+    await store.open()
+
+    try {
+      await fn(name, store, (message) => {
+        onProgress(parseInt((migrated / datastores.length) * 100), message)
+      })
+    } finally {
+      migrated++
+      store.close()
+    }
+  }
+
+  onProgress(100, `Migrated ${datastores.length} dbs`)
+}
+
+module.exports = {
+  version: 10,
+  description: 'Migrates datastore-level keys to binary',
+  migrate: (repoPath, repoOptions, onProgress) => {
+    return process(repoPath, repoOptions, onProgress, keysToBinary)
+  },
+  revert: (repoPath, repoOptions, onProgress) => {
+    return process(repoPath, repoOptions, onProgress, keysToStrings)
+  }
+}
+
+/**
+ * @typedef {Uint8Array|string} Key
+ * @typedef {Uint8Array} Value
+ * @typedef {{ type: 'del', key: Key } | { type: 'put', key: Key, value: Value }} Operation
+ *
+ * Uses the upgrade strategy from [email protected] - note we can't call the `.upgrade` command
+ * directly because it will be removed in [email protected] and we can't guarantee users will
+ * have migrated by then - e.g. they may jump from [email protected] straight to [email protected]
+ * so we have to duplicate the code here.
+ *
+ * @param {import('interface-datastore').Datastore} db
+ * @param {function (Key, Value): Operation[]} fn
+ */
+function withEach (db, fn) {
+  function batch (operations, next) {
+    const store = db.store('readwrite')
+    const transaction = store.transaction
+    let index = 0
+    let error
+
+    transaction.onabort = () => next(error || transaction.error || new Error('aborted by user'))
+    transaction.oncomplete = () => next()
+
+    function loop () {
+      var op = operations[index++]
+      var key = op.key
+
+      try {
+        var req = op.type === 'del' ? store.delete(key) : store.put(op.value, key)
+      } catch (err) {
+        error = err
+        transaction.abort()
+        return
+      }
+
+      if (index < operations.length) {
+        req.onsuccess = loop
+      }
+    }
+
+    loop()
+  }
+
+  return new Promise((resolve, reject) => {
+    const it = db.iterator()
+    // raw keys and values only
+    it._deserializeKey = it._deserializeValue = (data) => data
+    next()
+
+    function next () {
+      it.next((err, key, value) => {
+        if (err || key === undefined) {
+          it.end((err2) => {
+            if (err2) {
+              reject(err2)
+              return
+            }
+
+            resolve()
+          })
+
+          return
+        }
+
+        batch(fn(key, value), next)
+      })
+    }
+  })
+}

package.json

@@ -64,6 +64,8 @@
     "datastore-level": "^3.0.0",
     "it-all": "^1.0.2",
     "just-safe-set": "^2.1.0",
+    "level-5": "npm:level@^5.0.0",
+    "level-6": "npm:level@^6.0.0",
     "ncp": "^2.0.0",
     "rimraf": "^3.0.0",
     "sinon": "^9.0.2"

src/repo/version.js

@@ -4,6 +4,7 @@ const repoInit = require('./init')
 const { MissingRepoOptionsError, NotInitializedRepoError } = require('../errors')
 const { VERSION_KEY, createStore } = require('../utils')
 const uint8ArrayFromString = require('uint8arrays/from-string')
+const uint8ArrayToString = require('uint8arrays/to-string')
 
 exports.getVersion = getVersion
 
@@ -28,7 +29,14 @@ async function getVersion (path, repoOptions) {
   const store = createStore(path, 'root', repoOptions)
   await store.open()
 
-  const version = parseInt(await store.get(VERSION_KEY))
+  let version = await store.get(VERSION_KEY)
+
+  if (version instanceof Uint8Array) {
+    version = uint8ArrayToString(version)
+  }
+
+  version = parseInt(version)
+
   await store.close()
 
   return version

src/utils.js

@@ -1,11 +1,19 @@
 'use strict'
 
-const Key = require('interface-datastore').Key
+const {
+  Key,
+  Errors
+} = require('interface-datastore')
 const core = require('datastore-core')
 const ShardingStore = core.ShardingDatastore
 
-exports.CONFIG_KEY = new Key('/config')
-exports.VERSION_KEY = new Key('/version')
+/**
+ * @typedef {import('interface-datastore').Key} Key
+ * @typedef {import('interface-datastore').Datastore} Datastore
+ */
+
+const CONFIG_KEY = new Key('/config')
+const VERSION_KEY = new Key('/version')
 
 function getDatastoreAndOptions (name, options) {
   if (!options || !options.storageBackends) {
@@ -30,6 +38,94 @@ function getDatastoreAndOptions (name, options) {
   }
 }
 
+/**
+ * Level dbs wrap level dbs that wrap level dbs. Find a level-js
+ * instance in the chain if one exists.
+ *
+ * @param {Datastore} store
+ */
+function findLevelJs (store) {
+  let db = store
+
+  while (db.db || db.child) {
+    db = db.db || db.child
+
+    // `Level` is only present in the browser, in node it is LevelDOWN
+    if (db.type === 'level-js') {
+      return db
+    }
+  }
+}
+
+/**
+ * @param {Key} key
+ * @param {function (Key): Promise<boolean>} has
+ * @param {Datastore} store
+ */
+async function hasWithFallback (key, has, store) {
+  const result = await has(key)
+
+  if (result) {
+    return result
+  }
+
+  // Newer versions of level.js changed the key type from Uint8Array|string
+  // to Uint8Array  so fall back to trying Uint8Arrays if we are using level.js
+  // and the string version of the key did not work
+  const levelJs = findLevelJs(store)
+
+  if (!levelJs) {
+    return false
+  }
+
+  return new Promise((resolve, reject) => {
+    // drop down to IndexDB API, otherwise level-js will monkey around with the keys/values
+    const req = levelJs.store('readonly').get(key.toString())
+    req.transaction.onabort = () => {
+      reject(req.transaction.error)
+    }
+    req.transaction.oncomplete = () => {
+      resolve(Boolean(req.result))
+    }
+  })
+}
+
+/**
+ * @param {import('interface-datastore').Key} key
+ * @param {function (Key): Promise<Uint8Array>} get
+ * @param {function (Key): Promise<boolean>} has
+ * @param {import('interface-datastore').Datastore} store
+ */
+async function getWithFallback (key, get, has, store) {
+  if (await has(key)) {
+    return get(key)
+  }
+
+  // Newer versions of level.js changed the key type from Uint8Array|string
+  // to Uint8Array  so fall back to trying Uint8Arrays if we are using level.js
+  // and the string version of the key did not work
+  const levelJs = findLevelJs(store)
+
+  if (!levelJs) {
+    throw Errors.notFoundError()
+  }
+
+  return new Promise((resolve, reject) => {
+    // drop down to IndexDB API, otherwise level-js will monkey around with the keys/values
+    const req = levelJs.store('readonly').get(key.toString())
+    req.transaction.onabort = () => {
+      reject(req.transaction.error)
+    }
+    req.transaction.oncomplete = () => {
+      if (req.result) {
+        return resolve(req.result)
+      }
+
+      reject(Errors.notFoundError())
+    }
+  })
+}
+
 function createStore (location, name, options) {
   const { StorageBackend, storageOptions } = getDatastoreAndOptions(name, options)
 
@@ -43,14 +139,20 @@ function createStore (location, name, options) {
     store = new ShardingStore(store, new core.shard.NextToLast(2))
   }
 
+  // necessary since level-js@5 cannot read keys from level-js@4 and earlier
+  const originalGet = store.get.bind(store)
+  const originalHas = store.has.bind(store)
+  store.get = (key) => getWithFallback(key, originalGet, originalHas, store)
+  store.has = (key) => hasWithFallback(key, originalHas, store)
+
   return store
 }
 
-function containsIrreversibleMigration (from, to, migrations) {
-  return migrations
-    .filter(migration => migration.version > from && migration.version <= to)
-    .some(migration => migration.revert === undefined)
+module.exports = {
+  createStore,
+  hasWithFallback,
+  getWithFallback,
+  findLevelJs,
+  CONFIG_KEY,
+  VERSION_KEY
 }
-
-exports.createStore = createStore
-exports.containsIrreversibleMigration = containsIrreversibleMigration