From 0f1e00f397e76d1ac9cc924646b6e3cf3cdd87a2 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 21 Feb 2018 03:21:48 -0500 Subject: [PATCH] feat: jsipfs ls -r (Recursive list directory) (#1222) --- package.json | 4 +- src/cli/commands/file/ls.js | 2 + src/cli/commands/files.js | 6 ++- src/cli/commands/ls.js | 27 +++++----- src/core/components/files.js | 34 ++++++++---- src/http/api/resources/files.js | 10 ++-- test/cli/file.js | 6 ++- test/cli/files.js | 67 ------------------------ test/cli/ls.js | 92 +++++++++++++++++++++++++++++++++ 9 files changed, 150 insertions(+), 98 deletions(-) create mode 100644 test/cli/ls.js diff --git a/package.json b/package.json index 382d8f7326..ad1c74bbfc 100644 --- a/package.json +++ b/package.json @@ -106,9 +106,9 @@ "hapi": "^16.6.2", "hapi-set-header": "^1.0.2", "hoek": "^5.0.3", - "ipfs-api": "^18.0.0", - "ipfs-bitswap": "~0.19.0", "human-to-milliseconds": "^1.0.0", + "ipfs-api": "^18.1.1", + "ipfs-bitswap": "~0.19.0", "ipfs-block": "~0.6.1", "ipfs-block-service": "~0.13.0", "ipfs-multipart": "~0.1.0", diff --git a/src/cli/commands/file/ls.js b/src/cli/commands/file/ls.js index c952087b98..31abd73e60 100644 --- a/src/cli/commands/file/ls.js +++ b/src/cli/commands/file/ls.js @@ -11,6 +11,8 @@ module.exports = { handler (argv) { let path = argv.key + // `ipfs file ls` is deprecated. See https://ipfs.io/docs/commands/#ipfs-file-ls + print(`This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.`) argv.ipfs.ls(path, (err, links) => { if (err) { throw err diff --git a/src/cli/commands/files.js b/src/cli/commands/files.js index 09ca4de380..1e24b3c226 100644 --- a/src/cli/commands/files.js +++ b/src/cli/commands/files.js @@ -1,5 +1,8 @@ 'use strict' +const print = require('../utils').print +const lsCmd = require('./ls') + module.exports = { command: 'files ', @@ -8,9 +11,10 @@ module.exports = { builder (yargs) { return yargs .commandDir('files') + .command(lsCmd) }, handler (argv) { - console.log('Type `jsipfs bitswap --help` for more instructions') + print('Type `jsipfs files --help` for more instructions') } } diff --git a/src/cli/commands/ls.js b/src/cli/commands/ls.js index e7beecbaee..cf63dde2e1 100644 --- a/src/cli/commands/ls.js +++ b/src/cli/commands/ls.js @@ -14,6 +14,12 @@ module.exports = { type: 'boolean', default: false }, + r: { + alias: 'recursive', + desc: 'List subdirectories recursively', + type: 'boolean', + default: false + }, 'resolve-type': { desc: 'Resolve linked objects to find out their types. (not implemented yet)', type: 'boolean', @@ -27,7 +33,7 @@ module.exports = { path = path.replace('/ipfs/', '') } - argv.ipfs.ls(path, (err, links) => { + argv.ipfs.ls(path, { recursive: argv.recursive }, (err, links) => { if (err) { throw err } @@ -36,20 +42,17 @@ module.exports = { links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links) } - links = links.filter((link) => link.path !== path) - links.forEach((link) => { - if (link.type === 'dir') { - // directory: add trailing "/" - link.name = (link.name || '') + '/' - } - }) const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length)) const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) - links.forEach((file) => { - utils.print(utils.rightpad(file.hash, multihashWidth + 1) + - utils.rightpad(file.size || '', sizeWidth + 1) + - file.name) + links.forEach(link => { + const fileName = link.type === 'dir' ? `${link.name || ''}/` : link.name + const padding = link.depth - path.split('/').length + utils.print( + utils.rightpad(link.hash, multihashWidth + 1) + + utils.rightpad(link.size || '', sizeWidth + 1) + + ' '.repeat(padding) + fileName + ) }) }) } diff --git a/src/core/components/files.js b/src/core/components/files.js index b06c396745..d2afecf767 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -161,14 +161,20 @@ module.exports = function files (self) { return d } - function _lsPullStreamImmutable (ipfsPath) { + function _lsPullStreamImmutable (ipfsPath, options) { const path = normalizePath(ipfsPath) - const depth = path.split('/').length + const recursive = options && options.recursive + const pathDepth = path.split('/').length + const maxDepth = recursive ? global.Infinity : pathDepth + return pull( - exporter(ipfsPath, self._ipldResolver, { maxDepth: depth }), - pull.filter((node) => node.depth === depth), - pull.map((node) => { - node = Object.assign({}, node, { hash: toB58String(node.hash) }) + exporter(ipfsPath, self._ipldResolver, { maxDepth: maxDepth }), + pull.filter(node => + recursive ? node.depth >= pathDepth : node.depth === pathDepth + ), + pull.map(node => { + const cid = new CID(node.hash) + node = Object.assign({}, node, { hash: cid.toBaseEncodedString() }) delete node.content return node }) @@ -278,20 +284,26 @@ module.exports = function files (self) { return exporter(ipfsPath, self._ipldResolver) }, - lsImmutable: promisify((ipfsPath, callback) => { + lsImmutable: promisify((ipfsPath, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + pull( - _lsPullStreamImmutable(ipfsPath), + _lsPullStreamImmutable(ipfsPath, options), pull.collect((err, values) => { if (err) { - return callback(err) + callback(err) + return } callback(null, values) }) ) }), - lsReadableStreamImmutable: (ipfsPath) => { - return toStream.source(_lsPullStreamImmutable(ipfsPath)) + lsReadableStreamImmutable: (ipfsPath, options) => { + return toStream.source(_lsPullStreamImmutable(ipfsPath, options)) }, lsPullStreamImmutable: _lsPullStreamImmutable diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index 1349b46ca8..55c406d86c 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -272,13 +272,14 @@ exports.immutableLs = { handler: (request, reply) => { const key = request.pre.args.key const ipfs = request.server.app.ipfs + const recursive = request.query && request.query.recursive === 'true' - ipfs.ls(key, (err, files) => { + ipfs.ls(key, { recursive: recursive }, (err, files) => { if (err) { - reply({ + return reply({ Message: 'Failed to list dir: ' + err.message, Code: 0 - }).code(500) + }).code(500).takeover() } reply({ @@ -288,7 +289,8 @@ exports.immutableLs = { Name: file.name, Hash: file.hash, Size: file.size, - Type: toTypeCode(file.type) + Type: toTypeCode(file.type), + Depth: file.depth })) }] }) diff --git a/test/cli/file.js b/test/cli/file.js index 59ab531f6d..afe4d98db0 100644 --- a/test/cli/file.js +++ b/test/cli/file.js @@ -17,12 +17,16 @@ describe('file ls', () => runOnAndOff((thing) => { it('prints a filename', () => { return ipfs(`file ls ${file}`) - .then((out) => expect(out).to.eql(`${file}\n`)) + .then((out) => expect(out).to.eql( + `This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.\n` + + `${file}\n` + )) }) it('prints the filenames in a directory', () => { return ipfs(`file ls ${dir}`) .then((out) => expect(out).to.eql( + `This functionality is deprecated, and will be removed in future versions. If possible, please use 'ipfs ls' instead.\n` + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9\n' + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN\n' + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz\n' + diff --git a/test/cli/files.js b/test/cli/files.js index 9fa95a0401..b2d317abd3 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -296,73 +296,6 @@ describe('files', () => runOnAndOff((thing) => { }) }) - it('ls', function () { - this.timeout(20 * 1000) - - return ipfs('ls QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2') - .then((out) => { - expect(out).to.eql( - 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' + - 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + - 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore/\n' + - 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs/\n' + - 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n') - }) - }) - - it('ls -v', function () { - this.timeout(20 * 1000) - - return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2 -v') - .then((out) => { - expect(out).to.eql( - 'Hash Size Name\n' + - 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' + - 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + - 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore/\n' + - 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs/\n' + - 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n') - }) - }) - - it('ls ', function () { - this.timeout(20 * 1000) - - return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2/init-docs') - .then((out) => { - expect(out).to.eql( - 'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' + - 'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200 contact\n' + - 'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65 docs/\n' + - 'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322 help\n' + - 'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' + - 'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' + - 'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' + - 'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863 tour/\n') - }) - }) - - it('ls --help', function () { - this.timeout(20 * 1000) - - return ipfs('ls --help') - .then((out) => { - expect(out.split('\n').slice(1)).to.eql(['', - 'List files for the given directory', - '', - 'Options:', - ' --version Show version number [boolean]', - ' --silent Write no output [boolean] [default: false]', - ' --pass Pass phrase for the keys [string] [default: ""]', - ' --help Show help [boolean]', - ' -v, --headers Print table headers (Hash, Size, Name).', - ' [boolean] [default: false]', - ' --resolve-type Resolve linked objects to find out their types. (not', - ' implemented yet) [boolean] [default: false]', - '', '']) - }) - }) - it('get', function () { this.timeout(20 * 1000) diff --git a/test/cli/ls.js b/test/cli/ls.js new file mode 100644 index 0000000000..80b75bf342 --- /dev/null +++ b/test/cli/ls.js @@ -0,0 +1,92 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const runOnAndOff = require('../utils/on-and-off') + +describe('ls', () => runOnAndOff((thing) => { + let ipfs + + before(() => { + ipfs = thing.ipfs + return ipfs('files add -r test/fixtures/test-data/recursive-get-dir') + }) + + it('prints added files', function () { + this.timeout(20 * 1000) + return ipfs('ls QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2') + .then((out) => { + expect(out).to.eql( + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' + + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore/\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs/\n' + + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n' + ) + }) + }) + + it('prints nothing for non-existant hashes', function () { + // If the daemon is off, ls should fail + // If the daemon is on, ls should search until it hits a timeout + return Promise.race([ + ipfs.fail('ls QmYmW4HiZhotsoSqnv2o1oSssvkRM8b9RweBoH7ao5nki2'), + new Promise((res, rej) => setTimeout(res, 4000)) + ]) + .catch(() => expect.fail(0, 1, 'Should have thrown or timedout')) + }) + + it('adds a header, -v', function () { + this.timeout(20 * 1000) + return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2 -v') + .then((out) => { + expect(out).to.eql( + 'Hash Size Name\n' + + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' + + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore/\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs/\n' + + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n' + ) + }) + }) + + it('follows a path, /', function () { + this.timeout(20 * 1000) + + return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2/init-docs') + .then((out) => { + expect(out).to.eql( + 'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' + + 'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200 contact\n' + + 'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65 docs/\n' + + 'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322 help\n' + + 'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' + + 'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' + + 'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' + + 'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863 tour/\n' + ) + }) + }) + + it('recursively follows folders, -r', function () { + this.slow(2000) + this.timeout(20 * 1000) + + return ipfs('ls -r /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2/init-docs') + .then(out => { + expect(out).to.eql( + 'QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V 1688 about\n' + + 'QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y 200 contact\n' + + 'QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC 65 docs/\n' + + 'QmQN88TEidd3RY2u3dpib49fERTDfKtDpvxnvczATNsfKT 14 index\n' + + 'QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 322 help\n' + + 'QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha 1728 quick-start\n' + + 'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB 1102 readme\n' + + 'QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ 1027 security-notes\n' + + 'QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te 863 tour/\n' + + 'QmYE7xo6NxbHEVEHej1yzxijYaNY51BaeKxjXxn6Ssa6Bs 807 0.0-intro\n' + ) + }) + }) +}))