diff --git a/package.json b/package.json index 549858e93b..b282716f5f 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "dependencies": { "@nodeutils/defaults-deep": "^1.1.0", "async": "^2.6.1", + "async-iterator-to-pull-stream": "^1.1.0", "bignumber.js": "^8.0.2", "binary-querystring": "~0.1.2", "bl": "^3.0.0", @@ -114,7 +115,7 @@ "ipfs-unixfs": "~0.1.16", "ipfs-unixfs-exporter": "~0.36.1", "ipfs-unixfs-importer": "~0.38.5", - "ipld": "~0.21.1", + "ipld": "~0.22.0", "ipld-bitcoin": "~0.1.8", "ipld-dag-pb": "~0.15.3", "ipld-ethereum": "^2.0.1", diff --git a/src/core/components/dag.js b/src/core/components/dag.js index dc9ef2bdf5..967038b862 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -3,10 +3,12 @@ const promisify = require('promisify-es6') const CID = require('cids') const pull = require('pull-stream') +const iterToPull = require('async-iterator-to-pull-stream') const mapAsync = require('async/map') const setImmediate = require('async/setImmediate') const flattenDeep = require('lodash/flattenDeep') const errCode = require('err-code') +const multicodec = require('multicodec') module.exports = function dag (self) { return { @@ -25,21 +27,42 @@ module.exports = function dag (self) { } const optionDefaults = { - format: 'dag-cbor', - hashAlg: 'sha2-256' + format: multicodec.DAG_CBOR, + hashAlg: multicodec.SHA2_256 } - options = options.cid ? options : Object.assign({}, optionDefaults, options) + // The IPLD expects the format and hashAlg as constants + if (options.format && typeof options.format === 'string') { + const constantName = options.format.toUpperCase().replace(/-/g, '_') + options.format = multicodec[constantName] + } + if (options.hashAlg && typeof options.hashAlg === 'string') { + const constantName = options.hashAlg.toUpperCase().replace(/-/g, '_') + options.hashAlg = multicodec[constantName] + } - self._ipld.put(dagNode, options, (err, cid) => { - if (err) return callback(err) + options = options.cid ? options : Object.assign({}, optionDefaults, options) - if (options.preload !== false) { - self._preload(cid) - } + // js-ipld defaults to verion 1 CIDs. Hence set version 0 explicitly for + // dag-pb nodes + if (options.format === multicodec.DAG_PB && + options.hashAlg === multicodec.SHA2_256 && + options.version === undefined) { + options.version = 0 + } - callback(null, cid) - }) + self._ipld.put(dagNode, options.format, { + hashAlg: options.hashAlg, + cidVersion: options.version + }).then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } + return callback(null, cid) + }, + (error) => callback(error) + ) }), get: promisify((cid, path, options, callback) => { @@ -54,7 +77,7 @@ module.exports = function dag (self) { // Allow options in path position if (typeof path !== 'string') { options = path - path = null + path = undefined } else { options = {} } @@ -90,7 +113,26 @@ module.exports = function dag (self) { self._preload(cid) } - self._ipld.get(cid, path, options, callback) + if (path === undefined || path === '/') { + self._ipld.get(cid).then( + (value) => { + callback(null, { + value, + remainderPath: '' + }) + }, + (error) => callback(error) + ) + } else { + const result = self._ipld.resolve(cid, path) + const promisedValue = options.localResolve ? + result.first() : + result.last() + promisedValue.then( + (value) => callback(null, value), + (error) => callback(error) + ) + } }), tree: promisify((cid, path, options, callback) => { @@ -135,7 +177,7 @@ module.exports = function dag (self) { } pull( - self._ipld.treeStream(cid, path, options), + iterToPull(self._ipld.tree(cid, path, options)), pull.collect(callback) ) }), diff --git a/src/core/components/init.js b/src/core/components/init.js index 29cac89402..3d0890619d 100644 --- a/src/core/components/init.js +++ b/src/core/components/init.js @@ -11,6 +11,7 @@ const { DAGNode } = require('ipld-dag-pb') const UnixFs = require('ipfs-unixfs') +const multicodec = require('multicodec') const IPNS = require('../ipns') const OfflineDatastore = require('../ipns/routing/offline-datastore') @@ -132,9 +133,12 @@ module.exports = function init (self) { (cb) => DAGNode.create(new UnixFs('directory').marshal(), cb), (node, cb) => self.dag.put(node, { version: 0, - format: 'dag-pb', - hashAlg: 'sha2-256' - }, cb), + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256 + }).then( + (cid) => cb(null, cid), + (error) => cb(error) + ), (cid, cb) => self._ipns.initializeKeyspace(privateKey, cid.toBaseEncodedString(), cb) ], cb) } diff --git a/src/core/components/object.js b/src/core/components/object.js index 85a3137522..d33f1ffa09 100644 --- a/src/core/components/object.js +++ b/src/core/components/object.js @@ -9,6 +9,7 @@ const DAGNode = dagPB.DAGNode const DAGLink = dagPB.DAGLink const CID = require('cids') const mh = require('multihashes') +const multicodec = require('multicodec') const Unixfs = require('ipfs-unixfs') const errCode = require('err-code') @@ -87,19 +88,19 @@ module.exports = function object (self) { return cb(err) } - self._ipld.put(node, { - version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (err, cid) => { - if (err) return cb(err) - - if (options.preload !== false) { - self._preload(cid) - } - - cb(null, cid) - }) + self._ipld.put(node, multicodec.DAG_PB, { + cidVersion: 0, + hashAlg: multicodec.SHA2_256, + }).then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } + + cb(null, cid) + }, + (error) => cb(error) + ) }) } ], callback) @@ -137,21 +138,19 @@ module.exports = function object (self) { return callback(err) } - self._ipld.put(node, { - version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (err, cid) => { - if (err) { - return callback(err) - } - - if (options.preload !== false) { - self._preload(cid) - } + self._ipld.put(node, multicodec.DAG_PB, { + cidVersion: 0, + hashAlg: multicodec.SHA2_256, + }).then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } - callback(null, cid) - }) + callback(null, cid) + }, + (error) => callback(error) + ) }) }), put: promisify((obj, options, callback) => { @@ -200,21 +199,19 @@ module.exports = function object (self) { } function next () { - self._ipld.put(node, { - version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (err, cid) => { - if (err) { - return callback(err) - } - - if (options.preload !== false) { - self._preload(cid) - } + self._ipld.put(node, multicodec.DAG_PB, { + cidVersion: 0, + hashAlg: multicodec.SHA2_256, + }).then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } - callback(null, cid) - }) + callback(null, cid) + }, + (error) => callback(error) + ) } }), @@ -248,13 +245,10 @@ module.exports = function object (self) { self._preload(cid) } - self._ipld.get(cid, (err, result) => { - if (err) { - return callback(err) - } - - callback(null, result.value) - }) + self._ipld.get(cid).then( + (node) => callback(null, node), + (error) => callback(error) + ) }), data: promisify((multihash, options, callback) => { diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js index 31ecfffdf0..c178801c61 100644 --- a/src/core/components/pin-set.js +++ b/src/core/components/pin-set.js @@ -6,6 +6,7 @@ const protobuf = require('protons') const fnv1a = require('fnv1a') const varint = require('varint') const { DAGNode, DAGLink } = require('ipld-dag-pb') +const multicodec = require('multicodec') const someSeries = require('async/someSeries') const eachOfSeries = require('async/eachOfSeries') @@ -110,8 +111,8 @@ exports = module.exports = function (dag) { dag.put(rootNode, { version: 0, - format: 'dag-pb', - hashAlg: 'sha2-256', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false }, (err, cid) => { if (err) { return callback(err, cid) } @@ -194,8 +195,8 @@ exports = module.exports = function (dag) { const opts = { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false } diff --git a/src/core/components/pin.js b/src/core/components/pin.js index e74427a96a..b6b1b53439 100644 --- a/src/core/components/pin.js +++ b/src/core/components/pin.js @@ -15,6 +15,7 @@ const setImmediate = require('async/setImmediate') const { Key } = require('interface-datastore') const errCode = require('err-code') const multibase = require('multibase') +const multicodec = require('multicodec') const createPinSet = require('./pin-set') const { resolvePath } = require('../utils') @@ -104,8 +105,8 @@ module.exports = (self) => { if (err) { return cb(err) } dag.put(empty, { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false }, cb) }), @@ -116,8 +117,8 @@ module.exports = (self) => { root = node dag.put(root, { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false }, (err, cid) => { if (!err) { diff --git a/src/core/components/resolve.js b/src/core/components/resolve.js index 1ca80e8249..f863e75d1c 100644 --- a/src/core/components/resolve.js +++ b/src/core/components/resolve.js @@ -8,7 +8,7 @@ const CID = require('cids') const { cidToString } = require('../../utils/cid') module.exports = (self) => { - return promisify((name, opts, cb) => { + return promisify(async (name, opts, cb) => { if (typeof opts === 'function') { cb = opts opts = {} @@ -34,56 +34,26 @@ module.exports = (self) => { const path = split.slice(3).join('/') - resolve(cid, path, (err, res) => { - if (err) return cb(err) - const { cid, remainderPath } = res - cb(null, `/ipfs/${cidToString(cid, { base: opts.cidBase })}${remainderPath ? '/' + remainderPath : ''}`) - }) - }) - - // Resolve the given CID + path to a CID. - function resolve (cid, path, callback) { - let value, remainderPath - doUntil( - (cb) => { - self.block.get(cid, (err, block) => { - if (err) return cb(err) - - const r = self._ipld.resolvers[cid.codec] - - if (!r) { - return cb(new Error(`No resolver found for codec "${cid.codec}"`)) - } - - r.resolver.resolve(block.data, path, (err, result) => { - if (err) return cb(err) + const results = self._ipld.resolve(cid, path) + let value = cid + let remainderPath = path + try { + for await (const result of results) { + if (result.remainderPath === '') { + // Use values from previous iteration if the value isn't a CID + if (CID.isCID(result.value)) { value = result.value - remainderPath = result.remainderPath - cb() - }) - }) - }, - () => { - if (value && value['/']) { - // If we've hit a CID, replace the current CID. - cid = new CID(value['/']) - path = remainderPath - } else if (CID.isCID(value)) { - // If we've hit a CID, replace the current CID. - cid = value - path = remainderPath - } else { - // We've hit a value. Return the current CID and the remaining path. - return true + remainderPath = '' + } + break } - // Continue resolving unless the path is empty. - return !path || path === '/' - }, - (err) => { - if (err) return callback(err) - callback(null, { cid, remainderPath: path }) + value = result.value + remainderPath = result.remainderPath } - ) - } + } catch (error) { + return cb(error) + } + return cb(null, `/ipfs/${cidToString(value, { base: opts.cidBase })}${remainderPath ? '/' + remainderPath : ''}`) + }) } diff --git a/src/core/index.js b/src/core/index.js index 8248a21765..aaca8ba708 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -27,34 +27,34 @@ const mfsPreload = require('./mfs-preload') // All known (non-default) IPLD formats const IpldFormats = { - get 'bitcoin-block' () { + get [multicodec.BITCOIN_BOCK] () { return require('ipld-bitcoin') }, - get 'eth-account-snapshot' () { + get [multicodec.ETH_ACCOUNT_SNAPSHOT] () { return require('ipld-ethereum').ethAccountSnapshot }, - get 'eth-block' () { + get [multicodec.ETH_BLOCK] () { return require('ipld-ethereum').ethBlock }, - get 'eth-block-list' () { + get [multicodec.ETH_BLOCK_LIST] () { return require('ipld-ethereum').ethBlockList }, - get 'eth-state-trie' () { + get [multicodec.ETH_STATE_TRIE] () { return require('ipld-ethereum').ethStateTrie }, - get 'eth-storage-trie' () { + get [multicodec.ETH_STORAGE_TRIE] () { return require('ipld-ethereum').ethStorageTrie }, - get 'eth-tx' () { + get [multicodec.ETH_TX] () { return require('ipld-ethereum').ethTx }, - get 'eth-tx-trie' () { + get [multicodec.ETH_TX_TRIE] () { return require('ipld-ethereum').ethTxTrie }, - get 'git-raw' () { + get [multicodec.GIT_RAW] () { return require('ipld-git') }, - get 'zcash-block' () { + get [multicodec.ZCASH_BLOCK] () { return require('ipld-zcash') } } @@ -107,10 +107,13 @@ class IPFS extends EventEmitter { this._blockService = new BlockService(this._repo) this._ipld = new Ipld({ blockService: this._blockService, - loadFormat: (codec, callback) => { + loadFormat: async (codec) => { this.log('Loading IPLD format', codec) - if (IpldFormats[codec]) return callback(null, IpldFormats[codec]) - callback(new Error(`Missing IPLD format "${codec}"`)) + if (IpldFormats[codec]) { + return IpldFormats[codec] + } else { + throw new Error(`Missing IPLD format "${codec}"`) + } } }) this._preload = preload(this) diff --git a/src/http/api/resources/dag.js b/src/http/api/resources/dag.js index 325588f646..61c73d458f 100644 --- a/src/http/api/resources/dag.js +++ b/src/http/api/resources/dag.js @@ -6,6 +6,7 @@ const multipart = require('ipfs-multipart') const mh = require('multihashes') const Joi = require('joi') const multibase = require('multibase') +const multicodec = require('multicodec') const Boom = require('boom') const debug = require('debug') const { @@ -173,15 +174,15 @@ exports.put = { } } else { const { ipfs } = request.server.app - const codec = ipfs._ipld.resolvers[format] - if (!codec) { - throw Boom.badRequest(`Missing IPLD format "${request.query.format}"`) - } - - const deserialize = promisify(codec.util.deserialize) - - node = await deserialize(data) + // IPLD expects the format and hashAlg as constants + const codecConstant = format.toUpperCase().replace(/-/g, '_') + node = await ipfs._ipld._deserialize({ + cid: { + codec: multicodec[codecConstant] + }, + data: data + }) } return { @@ -242,28 +243,17 @@ exports.resolve = { let lastCid = ref let lastRemainderPath = path - while (true) { - const block = await ipfs.block.get(lastCid) - const codec = ipfs._ipld.resolvers[lastCid.codec] - - if (!codec) { - throw Boom.badRequest(`Missing IPLD format "${lastCid.codec}"`) - } - - const resolve = promisify(codec.resolver.resolve) - const res = await resolve(block.data, lastRemainderPath) - - if (!res.remainderPath) { - break - } - - lastRemainderPath = res.remainderPath + if (path) { + const result = ipfs._ipld.resolve(lastCid, path) + while (true) { + const resolveResult = (await result.next()).value + if (!CID.isCID(resolveResult.value)) { + break + } - if (!CID.isCID(res.value)) { - break + lastRemainderPath = resolveResult.remainderPath + lastCid = resolveResult.value } - - lastCid = res.value } return h.response({