Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: dont spawn node for local ops #110

Merged
merged 2 commits into from
Jun 4, 2019
Merged

fix: dont spawn node for local ops #110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a711d5
chore: remove unneeded asyncs
fsdiogo May 28, 2019
3b47df7
fix: dont spawn node for local ops
fsdiogo May 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 2 additions & 42 deletions src/core/commands/proxy.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,10 @@
'use strict'

const config = require('../config')
const startIpfs = require('./start-ipfs')
const startServer = require('./start-server')
const rewriteLockfile = require('./rewrite-lock-file')
const request = require('ipfs-registry-mirror-common/utils/retry-request')
const {
spawn
} = require('child_process')
const { spawn } = require('child_process')
const which = require('which-promise')
const timeout = require('ipfs-registry-mirror-common/utils/timeout-promise')

const cleanUpOps = []

Expand All @@ -28,44 +23,9 @@ process.on('SIGINT', cleanUp)
module.exports = async (options) => {
options = config(options)

const ipfs = await startIpfs(options)

cleanUpOps.push(() => {
return new Promise((resolve) => {
if (options.ipfs.node !== 'proc') {
return resolve()
}

ipfs.stop(() => {
console.info('👿 IPFS node stopped') // eslint-disable-line no-console
resolve()
})
})
})

console.info('🗂️ Loading registry index from', options.registry) // eslint-disable-line no-console

try {
const mirror = await request(Object.assign({}, options.request, {
uri: options.registry,
json: true
}))

console.info('☎️ Dialling registry mirror', mirror.ipfs.addresses.join(',')) // eslint-disable-line no-console

await timeout(
ipfs.api.swarm.connect(mirror.ipfs.addresses[0]),
options.registryConnectTimeout
)

console.info('📱️ Connected to registry') // eslint-disable-line no-console
} catch (error) {
console.info('📴 Not connected to registry') // eslint-disable-line no-console
}

console.info('👩‍🚀 Starting local proxy') // eslint-disable-line no-console

const server = await startServer(options, ipfs.api)
const server = await startServer(options)

cleanUpOps.push(() => {
return new Promise((resolve) => {
Expand Down
30 changes: 12 additions & 18 deletions src/core/commands/start-ipfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const spawn = (createArgs, spawnArgs = { init: true }) => {

const startIpfs = async (config) => {
if (config.ipfs.node === 'proc') {
console.info(`👿 Spawning an in-process IPFS node using repo at ${config.ipfs.repo}`) // eslint-disable-line no-console
console.info(`😈 Spawning an in-process IPFS node using repo at ${config.ipfs.repo}`) // eslint-disable-line no-console

const node = await spawn({
type: 'proc',
Expand All @@ -31,46 +31,40 @@ const startIpfs = async (config) => {
repoPath: config.ipfs.repo
})

return new Promise(async (resolve, reject) => {
try {
const initalise = promisify(node.init.bind(node))
const start = promisify(node.start.bind(node))
const initalise = promisify(node.init.bind(node))
const start = promisify(node.start.bind(node))

if (!node.initialized) {
await initalise()
}
if (!node.initialized) {
await initalise()
}

await start()
await start()

resolve(node)
} catch (error) {
reject(error)
}
})
return node
} else if (config.ipfs.node === 'disposable') {
console.info('👿 Spawning an in-process disposable IPFS node') // eslint-disable-line no-console
console.info('😈 Spawning an in-process disposable IPFS node') // eslint-disable-line no-console

return spawn({
type: 'proc',
exec: require('ipfs')
})
} else if (config.ipfs.node === 'js') {
console.info('👿 Spawning a js-IPFS node') // eslint-disable-line no-console
console.info('😈 Spawning a js-IPFS node') // eslint-disable-line no-console

return spawn({
type: 'js',
exec: await which('jsipfs')
})
} else if (config.ipfs.node === 'go') {
console.info('👿 Spawning a go-IPFS node') // eslint-disable-line no-console
console.info('😈 Spawning a go-IPFS node') // eslint-disable-line no-console

return spawn({
type: 'go',
exec: await which('ipfs')
})
}

console.info(`👿 Connecting to a remote IPFS node at ${config.ipfs.node}`) // eslint-disable-line no-console
console.info(`😈 Connecting to a remote IPFS node at ${config.ipfs.node}`) // eslint-disable-line no-console

return {
api: new IpfsApi(config.ipfs.node),
Expand Down
17 changes: 8 additions & 9 deletions src/core/commands/start-server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,21 @@ const favicon = require('ipfs-registry-mirror-common/handlers/favicon')
const root = require('../handlers/root')
const tarball = require('../handlers/tarball')
const manifest = require('../handlers/manifest')
const getIpfs = require('../middlewares/getIpfs')

const startServer = (config, ipfs) => {
const startServer = (config) => {
const app = express()

app.use(requestLog)

app.get('/favicon.ico', favicon(config, ipfs, app))
app.get('/favicon.png', favicon(config, ipfs, app))
app.get('/favicon.ico', favicon(config, app))
app.get('/favicon.png', favicon(config, app))

app.get('/', root(config, ipfs, app))
app.get('/', getIpfs(config), root(config, app))

// intercept requests for tarballs and manifests
app.get('/*.tgz', tarball(config, ipfs, app))
app.get('/*', manifest(config, ipfs, app))
app.get('/*.tgz', getIpfs(config), tarball(config, app))
app.get('/*', getIpfs(config), manifest(config, app))

// everything else should just proxy for the registry
const registry = proxy(config.registry, {
Expand All @@ -35,9 +36,7 @@ const startServer = (config, ipfs) => {

app.use(errorLog)

app.locals.ipfs = ipfs

return new Promise(async (resolve, reject) => {
return new Promise((resolve, reject) => {
const callback = once((error) => {
if (error) {
reject(error)
Expand Down
6 changes: 4 additions & 2 deletions src/core/handlers/manifest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,18 @@ const replaceTarballUrls = (pkg, config) => {
return pkg
}

module.exports = (config, ipfs, app) => {
module.exports = (config, app) => {
return async (request, response, next) => {
log(`Requested ${request.path}`)

let moduleName = sanitiseName(request.path)

log(`Loading manifest for ${moduleName}`)

const ipfs = response.locals.ipfs

try {
const manifest = await loadManifest(config, ipfs, moduleName)
const manifest = await loadManifest(config, ipfs.api, moduleName)

// because we start the server on a random high port, the previously stored
// manifests may have port numbers from the last time we ran, so overwrite
Expand Down
6 changes: 4 additions & 2 deletions src/core/handlers/tarball.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,18 @@ const path = require('path')
const loadTarball = require('ipfs-registry-mirror-common/utils/load-tarball')
const lol = require('ipfs-registry-mirror-common/utils/error-message')

module.exports = (config, ipfs, app) => {
module.exports = (config, app) => {
return async (request, response, next) => {
log(`Requested ${request.path}`)

let file = request.path

log(`Loading ${file}`)

const ipfs = response.locals.ipfs

try {
const readStream = await loadTarball(config, ipfs, file)
const readStream = await loadTarball(config, ipfs.api, file)

readStream.on('error', (error) => {
log(`Error loading ${file} - ${error}`)
Expand Down
77 changes: 77 additions & 0 deletions src/core/middlewares/getIpfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
'use strict'

const startIpfs = require('../commands/start-ipfs')
const request = require('ipfs-registry-mirror-common/utils/retry-request')
const timeout = require('ipfs-registry-mirror-common/utils/timeout-promise')

const cleanUpOps = []

const cleanUp = async () => {
Promise.all(
cleanUpOps.map(op => op())
)
.then(() => {
process.exit(0)
})
}

process.on('SIGTERM', cleanUp)
process.on('SIGINT', cleanUp)

const ipfsInstance = (function() {
let ipfs = null

return {
get: () => ipfs,
set: (instance) => { ipfs = instance }
}
}())

module.exports = (options) => {
return async (req, res, next) => {
if (ipfsInstance.get() !== null) {
res.locals.ipfs = ipfsInstance.get()
return next()
}

const ipfs = await startIpfs(options)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This does not return immediately so multiple parallel GET requests to the server cause multiple IPFS nodes to be created. Better to have res.locals.ipfs be a promise to resolve an IPFS node as multiple contexts can await on it and it'll only resolve once.

On that res.locals.ipfs would be better as req.app.ipfs so the single IPFS node is explicitly global to the app rather than storing it on res.locals which imply per-request scoping.

Copy link
Member

@achingbrain achingbrain Jun 4, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By which I mean, when I run this locally I see this sort of thing as requests from npm come in.

The rejected promises are caused by all the IPFS instances fighting over the locked ipfs-repo.

😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
😈 Spawning an in-process IPFS node using repo at /Users/alex/.jsipfs
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:5539) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 2)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 3)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 4)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 5)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 6)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 7)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 8)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 9)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 10)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)
(node:5539) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 11)
(node:5539) UnhandledPromiseRejectionWarning: Error: Lock file is already being held
    at options.fs.stat (/Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/proper-lockfile/lib/lockfile.js:68:47)
    at /Users/alex/Documents/Workspaces/ipfs-shipyard/npm-on-ipfs/node_modules/graceful-fs/polyfills.js:285:20
    at FSReqWrap.oncomplete (fs.js:154:5)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Anyway, I fixed it in #111 but can't push to the incoming branch this PR is based on so have all the changes there.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, that's weird, I don't think that happened to me 🤔

You're right, that is a better approach 👍


cleanUpOps.push(() => {
return new Promise((resolve) => {
if (options.ipfs.node !== 'proc') {
return resolve()
}

ipfs.stop(() => {
console.info('😈 IPFS node stopped') // eslint-disable-line no-console
ipfsInstance.set(null)
resolve()
})
})
})

console.info('🗂️ Loading registry index from', options.registry) // eslint-disable-line no-console

try {
const mirror = await request(Object.assign({}, options.request, {
uri: options.registry,
json: true
}))

console.info('☎️ Dialling registry mirror', mirror.ipfs.addresses.join(',')) // eslint-disable-line no-console

await timeout(
ipfs.api.swarm.connect(mirror.ipfs.addresses[0]),
options.registryConnectTimeout
)

console.info('📱️ Connected to registry') // eslint-disable-line no-console
} catch (error) {
console.info('📴 Not connected to registry') // eslint-disable-line no-console
}

ipfsInstance.set(ipfs)
res.locals.ipfs = ipfs
next()
}
}