From c3823611b9bdfcb66ecfc4d1b2fc53a19faca90b Mon Sep 17 00:00:00 2001 From: "Nicholas J. Paterno" Date: Thu, 23 Jan 2020 19:27:23 -0500 Subject: [PATCH 1/2] Remove extra graceful shutdown config Fixes an issue introduced where graceful shutdown will wait for all files to cleanup --- lib/database/filedown.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/database/filedown.js b/lib/database/filedown.js index 5483167286..77a6219e43 100644 --- a/lib/database/filedown.js +++ b/lib/database/filedown.js @@ -4,7 +4,6 @@ var async = require("async"); var fs = require("fs"); var path = require("path"); var tmp = require("tmp"); -tmp.setGracefulCleanup(); util.inherits(FileDown, AbstractLevelDOWN); From 80df7f8090eaaf1947738bf16b94d7046758f0b2 Mon Sep 17 00:00:00 2001 From: David Murdoch Date: Fri, 24 Jan 2020 16:34:58 -0500 Subject: [PATCH 2/2] Fix issue where tmp has to clean up too many files --- lib/database/filedown.js | 60 ++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/lib/database/filedown.js b/lib/database/filedown.js index 77a6219e43..637d7e0f35 100644 --- a/lib/database/filedown.js +++ b/lib/database/filedown.js @@ -36,6 +36,34 @@ const accessQueue = { }, cache: {} }; + +const fds = new Set(); +const cleanup = (exit) => { + try { + fds.forEach((fdPath) => { + const [fd, path] = fdPath; + try { + fs.closeSync(fd); + } catch (e) { + // ignore + } finally { + try { + fs.unlinkSync(path); + } catch (e) { + // ignore + } + } + }); + fds.clear(); + } finally { + if (exit) { + process.exit(0); + } + } +}; +process.on("SIGINT", cleanup); +process.on("exit", () => cleanup(false)); + FileDown.prototype._put = function(key, value, options, callback) { const lKey = path.join(this.location, key); // This fixes an issue caused by writing AND reading the same key multiple times @@ -51,46 +79,38 @@ FileDown.prototype._put = function(key, value, options, callback) { // leveldb implementation that doesn't use a separate file for every key Soon(TM). accessQueue.execute(lKey, () => { // get a tmp file to write the contents to... - tmp.file((err, path, fd, cleanupTmpFile) => { + tmp.file({ keep: true }, (err, path, fd, cleanupTmpFile) => { if (err) { callback(err); accessQueue.next(lKey); return; } + const pair = [fd, path]; + fds.add(pair); + const cleanupAndCallback = (err) => { + err && cleanupTmpFile(); + fds.delete(pair); + callback(err); + accessQueue.next(lKey); + }; // write the value to our temporary file fs.writeFile(fd, value, "utf8", (err) => { if (err) { - cleanupTmpFile(); - callback(err); - accessQueue.next(lKey); + cleanupAndCallback(err); return; } // It worked! Move the temporary file to its final destination fs.rename(path, lKey, (err) => { if (err) { - cleanupTmpFile(); - callback(err); - accessQueue.next(lKey); + cleanupAndCallback(err); return; } // make sure we close this file descriptor now that the file is no // longer "temporary" (because we successfully moved it) - fs.close(fd, (err) => { - if (err) { - // at this point things seem to have worked, but juuuussttt in - // case `fs.close` fails, let's log some info so we can try to - // debug it - console.warn("An unexpected file descriptor close error occured: ", err); - cleanupTmpFile(); - } - callback(); - - // if there is more work to be done on this key, do it. - accessQueue.next(lKey); - }); + fs.close(fd, () => cleanupAndCallback()); }); }); });