feat: support --raw-leaves (#1454)

Also updates pinning to support CIDv1

package.json

@@ -108,11 +108,11 @@
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
     "ipfs-http-response": "~0.1.2",
-    "ipfs-mfs": "~0.1.0",
+    "ipfs-mfs": "~0.2.2",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.22.1",
     "ipfs-unixfs": "~0.1.15",
-    "ipfs-unixfs-engine": "~0.30.0",
+    "ipfs-unixfs-engine": "~0.31.3",
     "ipld": "~0.17.3",
     "ipld-dag-cbor": "~0.12.1",
     "ipld-dag-pb": "~0.14.5",

src/cli/commands/files/add.js

@@ -145,12 +145,13 @@ module.exports = {
     },
     'raw-leaves': {
       type: 'boolean',
-      default: undefined,
+      default: false,
       describe: 'Use raw blocks for leaf nodes. (experimental)'
     },
     'cid-version': {
       type: 'integer',
-      describe: 'Cid version. Non-zero value will change default of \'raw-leaves\' to true. (experimental)'
+      describe: 'CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)',
+      default: 0
     },
     hash: {
       type: 'string',
@@ -197,34 +198,6 @@ module.exports = {
       pin: argv.pin
     }
 
-    // Temporary restriction on raw-leaves:
-    // When cid-version=1 then raw-leaves MUST be present and false.
-    //
-    // This is because raw-leaves is not yet implemented in js-ipfs,
-    // and go-ipfs changes the value of raw-leaves to true when
-    // cid-version > 0 unless explicitly set to false.
-    //
-    // This retains feature parity without having to implement raw-leaves.
-    if (options.cidVersion > 0 && options.rawLeaves !== false) {
-      throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0')
-    }
-
-    // Temporary restriction on raw-leaves:
-    // When hash != undefined then raw-leaves MUST be present and false.
-    //
-    // This is because raw-leaves is not yet implemented in js-ipfs,
-    // and go-ipfs changes the value of raw-leaves to true when
-    // hash != undefined unless explicitly set to false.
-    //
-    // This retains feature parity without having to implement raw-leaves.
-    if (options.hash && options.rawLeaves !== false) {
-      throw new Error('Implied argument raw-leaves must be passed and set to false when hash argument is specified')
-    }
-
-    if (options.rawLeaves) {
-      throw new Error('Not implemented: raw-leaves')
-    }
-
     if (options.enableShardingExperiment && utils.isDaemonOn()) {
       throw new Error('Error: Enabling the sharding experiment should be done on the daemon')
     }

src/core/components/pin.js

@@ -4,7 +4,6 @@
 const promisify = require('promisify-es6')
 const { DAGNode, DAGLink } = require('ipld-dag-pb')
 const CID = require('cids')
-const multihashes = require('multihashes')
 const async = require('async')
 const { Key } = require('interface-datastore')
 
@@ -34,9 +33,9 @@ module.exports = (self) => {
   let recursivePins = new Set()
 
   const directKeys = () =>
-    Array.from(directPins).map(key => multihashes.fromB58String(key))
+    Array.from(directPins).map(key => new CID(key).buffer)
   const recursiveKeys = () =>
-    Array.from(recursivePins).map(key => multihashes.fromB58String(key))
+    Array.from(recursivePins).map(key => new CID(key).buffer)
 
   function getIndirectKeys (callback) {
     const indirectKeys = new Set()

src/core/utils.js

@@ -1,9 +1,9 @@
 'use strict'
 
-const multihashes = require('multihashes')
 const promisify = require('promisify-es6')
 const map = require('async/map')
 const isIpfs = require('is-ipfs')
+const CID = require('cids')
 
 exports.OFFLINE_ERROR = 'This command must be run in online mode. Try running \'ipfs daemon\' first.'
 
@@ -61,12 +61,15 @@ const resolvePath = promisify(function (objectAPI, ipfsPaths, callback) {
 
   map(ipfsPaths, (path, cb) => {
     if (typeof path !== 'string') {
+      let cid
+
       try {
-        multihashes.validate(path)
+        cid = new CID(path)
       } catch (err) {
         return cb(err)
       }
-      return cb(null, path)
+
+      return cb(null, cid.buffer)
     }
 
     let parsedPath
@@ -76,10 +79,10 @@ const resolvePath = promisify(function (objectAPI, ipfsPaths, callback) {
       return cb(err)
     }
 
-    const rootHash = multihashes.fromB58String(parsedPath.hash)
+    const rootHash = new CID(parsedPath.hash)
     const rootLinks = parsedPath.links
     if (!rootLinks.length) {
-      return cb(null, rootHash)
+      return cb(null, rootHash.buffer)
     }
 
     objectAPI.get(rootHash, follow.bind(null, rootLinks))

src/http/api/resources/files.js

@@ -153,20 +153,8 @@ exports.add = {
   validate: {
     query: Joi.object()
       .keys({
-        'cid-version': Joi.number().integer().min(0).max(1),
-        // Temporary restriction on raw-leaves:
-        // When cid-version=1 then raw-leaves MUST be present and false.
-        //
-        // This is because raw-leaves is not yet implemented in js-ipfs,
-        // and go-ipfs changes the value of raw-leaves to true when
-        // cid-version > 0 unless explicitly set to false.
-        //
-        // This retains feature parity without having to implement raw-leaves.
-        'raw-leaves': Joi.boolean().when('cid-version', {
-          is: 1,
-          then: Joi.boolean().valid(false).required(),
-          otherwise: Joi.boolean().valid(false)
-        }),
+        'cid-version': Joi.number().integer().min(0).max(1).default(0),
+        'raw-leaves': Joi.boolean(),
         'only-hash': Joi.boolean(),
         pin: Joi.boolean().default(true),
         'wrap-with-directory': Joi.boolean()

test/cli/files.js

@@ -205,56 +205,34 @@ describe('files', () => runOnAndOff((thing) => {
     })
   })
 
-  // Temporarily expect to fail as raw-leaves not yet implemented.
-  //
-  // When cid-version=1 then raw-leaves MUST be present and false.
-  //
-  // This is because raw-leaves is not yet implemented in js-ipfs,
-  // and go-ipfs changes the value of raw-leaves to true when
-  // cid-version > 0 unless explicitly set to false.
-  //
-  // This retains feature parity without having to implement raw-leaves.
   it('add with cid-version=1', function () {
     this.timeout(30 * 1000)
 
-    return new Promise((resolve, reject) => {
-      ipfs('add src/init-files/init-docs/readme --cid-version=1')
-        .then(() => reject(new Error('Raw leaves not expected to be implemented')))
-        .catch((err) => {
-          expect(err).to.exist()
-          resolve()
-        })
-    })
+    return ipfs('add src/init-files/init-docs/readme --cid-version=1')
+      .then((out) => {
+        expect(out)
+          .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n')
+      })
   })
 
-  // TODO: this test is failing, @alanshaw?
-  it.skip('add with cid-version=1 and raw-leaves=false', () => {
-    return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false').then((out) => {
-      expect(out)
-        .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n')
-    })
+  it('add with cid-version=1 and raw-leaves=false', function () {
+    this.timeout(30 * 1000)
+
+    return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false')
+      .then((out) => {
+        expect(out)
+          .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n')
+      })
   })
 
-  // Temporarily expect to fail as raw-leaves not yet implemented
-  //
-  // When cid-version=1 then raw-leaves MUST be present and false.
-  //
-  // This is because raw-leaves is not yet implemented in js-ipfs,
-  // and go-ipfs changes the value of raw-leaves to true when
-  // cid-version > 0 unless explicitly set to false.
-  //
-  // This retains feature parity without having to implement raw-leaves.
   it('add with cid-version=1 and raw-leaves=true', function () {
     this.timeout(30 * 1000)
 
-    return new Promise((resolve, reject) => {
-      ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true')
-        .then(() => reject(new Error('Raw leaves not expected to be implemented')))
-        .catch((err) => {
-          expect(err).to.exist()
-          resolve()
-        })
-    })
+    return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true')
+      .then((out) => {
+        expect(out)
+          .to.eql('added zdj7WiLc855B1KPRgV7Fh8ivjuAhePE1tuJafmxH5HmmSjqaD readme\n')
+      })
   })
 
   it('add --quiet', function () {