diff --git a/packages/interface-ipfs-core/package.json b/packages/interface-ipfs-core/package.json index 645fafca50..4719dbe0d5 100644 --- a/packages/interface-ipfs-core/package.json +++ b/packages/interface-ipfs-core/package.json @@ -94,7 +94,7 @@ "libp2p-websockets": "^0.16.2", "merge-options": "^3.0.4", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "nanoid": "^3.1.23", "p-map": "^4.0.0", "p-retry": "^4.5.0", diff --git a/packages/ipfs-cli/package.json b/packages/ipfs-cli/package.json index 095f27b6db..a17200ccba 100644 --- a/packages/ipfs-cli/package.json +++ b/packages/ipfs-cli/package.json @@ -86,7 +86,7 @@ "mafmt": "^10.0.0", "multiaddr": "^10.0.0", "multiaddr-to-uri": "^8.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "parse-duration": "^1.0.0", "pretty-bytes": "^5.4.1", "progress": "^2.0.3", diff --git a/packages/ipfs-core-types/package.json b/packages/ipfs-core-types/package.json index 532601e5d8..c824545e59 100644 --- a/packages/ipfs-core-types/package.json +++ b/packages/ipfs-core-types/package.json @@ -45,7 +45,7 @@ "dependencies": { "interface-datastore": "^6.0.2", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1", diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index b91cfaca45..8394c2d49b 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -129,7 +129,7 @@ "merge-options": "^3.0.4", "multiaddr": "^10.0.0", "multiaddr-to-uri": "^8.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "nanoid": "^3.1.23", "parse-duration": "^1.0.0", "timeout-abort-controller": "^3.0.0", diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 6bd388ed3b..f6136830e4 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -124,7 +124,7 @@ "mortice": "^2.0.0", "multiaddr": "^10.0.0", "multiaddr-to-uri": "^8.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "pako": "^1.0.2", "parse-duration": "^1.0.0", "peer-id": "^0.16.0", diff --git a/packages/ipfs-core/src/components/dag/export.js b/packages/ipfs-core/src/components/dag/export.js index 286d40c3a9..9d559bf7b5 100644 --- a/packages/ipfs-core/src/components/dag/export.js +++ b/packages/ipfs-core/src/components/dag/export.js @@ -1,11 +1,11 @@ import { CID } from 'multiformats/cid' import { createUnsafe } from 'multiformats/block' -import { base58btc } from 'multiformats/bases/base58' import { CarWriter } from '@ipld/car/writer' import { withTimeoutOption } from 'ipfs-core-utils/with-timeout-option' import debug from 'debug' import * as raw from 'multiformats/codecs/raw' import * as json from 'multiformats/codecs/json' +import { walk } from 'multiformats/traversal' const log = debug('ipfs:components:dag:import') @@ -23,6 +23,11 @@ const NO_LINKS_CODECS = [ * @typedef {import('ipfs-core-types/src/utils').AbortOptions} AbortOptions */ +/** + * @template T + * @typedef {import('multiformats/block').Block} Block + */ + /** * @param {Object} config * @param {IPFSRepo} config.repo @@ -53,12 +58,11 @@ export function createExport ({ repo, preload, codecs }) { let err = null ;(async () => { try { - await traverseWrite( - repo, - { signal: options.signal, timeout: options.timeout }, - cid, - writer, - codecs) + const load = makeLoader(repo, writer, { + signal: options.signal, + timeout: options.timeout + }, codecs) + await walk({ cid, load }) } catch (/** @type {any} */ e) { err = e } finally { @@ -81,52 +85,30 @@ export function createExport ({ repo, preload, codecs }) { } /** + * @template T * @param {IPFSRepo} repo - * @param {AbortOptions} options - * @param {CID} cid * @param {BlockWriter} writer + * @param {AbortOptions} options * @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs - * @param {Set} seen - * @returns {Promise} + * @returns {(cid:CID)=>Promise|null>} */ -async function traverseWrite (repo, options, cid, writer, codecs, seen = new Set()) { - const b58Cid = cid.toString(base58btc) - if (seen.has(b58Cid)) { - return - } +function makeLoader (repo, writer, options, codecs) { + return async (cid) => { + const codec = await codecs.getCodec(cid.code) - const block = await getBlock(repo, options, cid, codecs) + if (!codec) { + throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`) + } - log(`Adding block ${cid} to car`) - await writer.put(block) - seen.add(b58Cid) + const bytes = await repo.blocks.get(cid, options) - // recursive traversal of all links - for (const link of block.links) { - await traverseWrite(repo, options, link, writer, codecs, seen) - } -} + log(`Adding block ${cid} to car`) + await writer.put({ cid, bytes }) -/** - * @param {IPFSRepo} repo - * @param {AbortOptions} options - * @param {CID} cid - * @param {import('ipfs-core-utils/multicodecs').Multicodecs} codecs - * @returns {Promise<{cid:CID, bytes:Uint8Array, links:CID[]}>} - */ -async function getBlock (repo, options, cid, codecs) { - const bytes = await repo.blocks.get(cid, options) - - /** @type {CID[]} */ - let links = [] - const codec = await codecs.getCodec(cid.code) - - if (codec) { - const block = createUnsafe({ bytes, cid, codec }) - links = [...block.links()].map((l) => l[1]) - } else if (!NO_LINKS_CODECS.includes(cid.code)) { - throw new Error(`Can't decode links in block with codec 0x${cid.code.toString(16)} to form complete DAG`) - } + if (NO_LINKS_CODECS.includes(cid.code)) { + return null // skip this block, no need to look inside + } - return { cid, bytes, links } + return createUnsafe({ bytes, cid, codec }) + } } diff --git a/packages/ipfs-grpc-client/package.json b/packages/ipfs-grpc-client/package.json index 30770dbad5..4ae4879194 100644 --- a/packages/ipfs-grpc-client/package.json +++ b/packages/ipfs-grpc-client/package.json @@ -63,7 +63,7 @@ "it-first": "^1.0.4", "it-pushable": "^1.4.2", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "p-defer": "^3.0.0", "protobufjs": "^6.10.2", "wherearewe": "^1.0.0", diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 4470a88d4c..f3dcb13b1c 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -68,7 +68,7 @@ "it-last": "^1.0.4", "merge-options": "^3.0.4", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "parse-duration": "^1.0.0", "stream-to-it": "^0.2.2", "uint8arrays": "^3.0.0" diff --git a/packages/ipfs-http-gateway/package.json b/packages/ipfs-http-gateway/package.json index 3044d5df39..d8974b67f6 100644 --- a/packages/ipfs-http-gateway/package.json +++ b/packages/ipfs-http-gateway/package.json @@ -68,7 +68,7 @@ "it-last": "^1.0.4", "it-to-stream": "^1.0.0", "joi": "^17.2.1", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "uint8arrays": "^3.0.0", "uri-to-multiaddr": "^6.0.0" }, diff --git a/packages/ipfs-http-server/package.json b/packages/ipfs-http-server/package.json index 7e2f530281..3f61adf883 100644 --- a/packages/ipfs-http-server/package.json +++ b/packages/ipfs-http-server/package.json @@ -71,7 +71,7 @@ "joi": "^17.2.1", "just-safe-set": "^2.2.1", "multiaddr": "^10.0.0", - "multiformats": "^9.4.13", + "multiformats": "^9.5.1", "parse-duration": "^1.0.0", "stream-to-it": "^0.2.2", "timeout-abort-controller": "^3.0.0", diff --git a/packages/ipfs-message-port-client/package.json b/packages/ipfs-message-port-client/package.json index e53ef465c8..7ba0e2056c 100644 --- a/packages/ipfs-message-port-client/package.json +++ b/packages/ipfs-message-port-client/package.json @@ -50,7 +50,7 @@ "ipfs-message-port-protocol": "^0.11.0", "ipfs-unixfs": "^6.0.3", "it-peekable": "^1.0.2", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1", diff --git a/packages/ipfs-message-port-protocol/package.json b/packages/ipfs-message-port-protocol/package.json index 8eeedbf1a8..f3ff195b13 100644 --- a/packages/ipfs-message-port-protocol/package.json +++ b/packages/ipfs-message-port-protocol/package.json @@ -75,7 +75,7 @@ }, "dependencies": { "ipfs-core-types": "^0.10.0", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1", diff --git a/packages/ipfs-message-port-server/package.json b/packages/ipfs-message-port-server/package.json index 0767101ea9..6c56defe33 100644 --- a/packages/ipfs-message-port-server/package.json +++ b/packages/ipfs-message-port-server/package.json @@ -71,7 +71,7 @@ "ipfs-core-types": "^0.10.0", "ipfs-message-port-protocol": "^0.11.0", "it-all": "^1.0.4", - "multiformats": "^9.4.13" + "multiformats": "^9.5.1" }, "devDependencies": { "aegir": "^36.0.1",