From 9c04dbb6410a07f49410fd7d4188e2a9d890dd2d Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Fri, 22 Jun 2018 02:00:30 -0400 Subject: [PATCH] feat: add cache pruning - Add `prune` configuration information to README Prune, by default, caches that are older than 2 days after accumulating 50 megabytes of cache. --- .travis.yml | 8 +- README.md | 22 +++ index.js | 9 + lib/ChalkLoggerPlugin.js | 9 + lib/SystemPruneCaches.js | 163 ++++++++++++++++++ lib/util/log-messages.js | 29 ++++ tests/fixtures/hard-source-prune/config-hash | 1 + tests/fixtures/hard-source-prune/fib/index.js | 3 + tests/fixtures/hard-source-prune/index.js | 3 + .../hard-source-prune/webpack.config.js | 27 +++ tests/hard-source.js | 9 + 11 files changed, 279 insertions(+), 4 deletions(-) create mode 100644 lib/SystemPruneCaches.js create mode 100644 tests/fixtures/hard-source-prune/config-hash create mode 100644 tests/fixtures/hard-source-prune/fib/index.js create mode 100644 tests/fixtures/hard-source-prune/index.js create mode 100644 tests/fixtures/hard-source-prune/webpack.config.js diff --git a/.travis.yml b/.travis.yml index ed570133..ed4d84b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,15 +10,15 @@ matrix: - node_js: node env: NPM_SCRIPT=commitlint-travis - node_js: node - env: NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0" + env: NPM_SCRIPT=test NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0" - node_js: 8 - env: NPM_INSTALL_EXTRA="webpack@3 file-loader@0.11 html-webpack-plugin@2.22.0" + env: NPM_SCRIPT=test NPM_INSTALL_EXTRA="webpack@3 file-loader@0.11 html-webpack-plugin@2.22.0" - node_js: 8 - env: NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0" + env: NPM_SCRIPT=test NPM_INSTALL_EXTRA="webpack@4 file-loader@1 html-webpack-plugin@3.2.0" before_script: - npm install ${NPM_INSTALL_EXTRA} -script: npm run ${NPM_SCRIPT:test} +script: npm run ${NPM_SCRIPT} cache: directories: - node_modules diff --git a/README.md b/README.md index ec57fdd0..908ffb7c 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,16 @@ new HardSourceWebpackPlugin({ // 'debug', 'log', 'info', 'warn', or 'error'. level: 'debug', }, + // Clean up large, old caches automatically. + cachePrune: { + // Caches younger than `maxAge` are not considered for deletion. They must + // be at least this (default: 2 days) old in milliseconds. + maxAge: 2 * 24 * 60 * 60 * 1000, + // All caches together must be larger than `sizeThreshold` before any + // caches will be deleted. Together they must be at least this + // (default: 50 MB) big in bytes. + sizeThreshold: 50 * 1024 * 1024 + }, }), ``` @@ -146,6 +156,18 @@ The level of log messages to report down to. Defaults to 'debug' when mode is 'n For example 'debug' reports all messages while 'warn' reports warn and error level messages. +### `cachePrune` + +`hard-source` caches are by default created when the webpack configuration changes. Each cache holds a copy of all the data to create a build so they can become quite large. Once a cache is considered "old enough" that it is unlikely to be reused `hard-source` will delete it to free up space automatically. + +#### `maxAge` + +Caches older than `maxAge` in milliseconds are considered for automatic deletion. + +#### `sizeThreshold` + +For caches to be deleted, all of them together must total more than this threshold. + ## Troubleshooting ### Configuration changes are not being detected diff --git a/index.js b/index.js index 0c45dae7..36fae67f 100644 --- a/index.js +++ b/index.js @@ -241,6 +241,15 @@ class HardSourceWebpackPlugin { 'relativeHelpers', ]); + if (configHashInDirectory) { + const PruneCachesSystem = require('./lib/SystemPruneCaches'); + + new PruneCachesSystem( + path.dirname(cacheDirPath), + options.cachePrune, + ).apply(compiler); + } + function runReadOrReset(_compiler) { logger.unlock(); diff --git a/lib/ChalkLoggerPlugin.js b/lib/ChalkLoggerPlugin.js index 56e65d37..b327a1e0 100644 --- a/lib/ChalkLoggerPlugin.js +++ b/lib/ChalkLoggerPlugin.js @@ -27,6 +27,15 @@ const messages = { short: value => `Reading from cache ${value.data.configHash.substring(0, 8)}...`, }, + 'caches--delete-old': { + short: value => + `Deleted ${value.data.deletedSizeMB} MB. Using ${ + value.data.sizeMB + } MB of disk space.`, + }, + 'caches--keep': { + short: value => `Using ${value.data.sizeMB} MB of disk space.`, + }, 'environment--inputs': { short: value => `Tracking node dependencies with: ${value.data.inputs.join(', ')}.`, diff --git a/lib/SystemPruneCaches.js b/lib/SystemPruneCaches.js new file mode 100644 index 00000000..0d8f217a --- /dev/null +++ b/lib/SystemPruneCaches.js @@ -0,0 +1,163 @@ +const { readdir: _readdir, stat: _stat } = require('fs'); +const { basename, join } = require('path'); + +const _rimraf = require('rimraf'); + +const logMessages = require('./util/log-messages'); +const pluginCompat = require('./util/plugin-compat'); +const promisify = require('./util/promisify'); + +const readdir = promisify(_readdir); +const rimraf = promisify(_rimraf); +const stat = promisify(_stat); + +const directorySize = async dir => { + const _stat = await stat(dir); + if (_stat.isFile()) { + return _stat.size; + } + + if (_stat.isDirectory()) { + const names = await readdir(dir); + let size = 0; + for (const name of names) { + size += await directorySize(join(dir, name)); + } + return size; + } + + return 0; +}; + +class CacheInfo { + constructor(id = '') { + this.id = id; + this.lastModified = 0; + this.size = 0; + } + + static async fromDirectory(dir) { + const info = new CacheInfo(basename(dir)); + info.lastModified = new Date( + (await stat(join(dir, 'stamp'))).mtime, + ).getTime(); + info.size = await directorySize(dir); + return info; + } + + static async fromDirectoryChildren(dir) { + const children = []; + const names = await readdir(dir); + for (const name of names) { + children.push(await CacheInfo.fromDirectory(join(dir, name))); + } + return children; + } +} + +// Compilers for webpack with multiple parallel configurations might try to +// delete caches at the same time. Mutex lock the process of pruning to keep +// from multiple pruning runs from colliding with each other. +let deleteLock = null; + +class PruneCachesSystem { + constructor(cacheRoot, options = {}) { + this.cacheRoot = cacheRoot; + + this.options = Object.assign( + { + // Caches younger than `maxAge` are not considered for deletion. They + // must be at least this (default: 2 days) old in milliseconds. + maxAge: 2 * 24 * 60 * 60 * 1000, + // All caches together must be larger than `sizeThreshold` before any + // caches will be deleted. Together they must be at least this + // (default: 50 MB) big in bytes. + sizeThreshold: 50 * 1024 * 1024, + }, + options, + ); + } + + apply(compiler) { + const compilerHooks = pluginCompat.hooks(compiler); + + const deleteOldCaches = async () => { + while (deleteLock !== null) { + await deleteLock; + } + + let resolveLock; + + let infos; + try { + deleteLock = new Promise(resolve => { + resolveLock = resolve; + }); + + infos = await CacheInfo.fromDirectoryChildren(this.cacheRoot); + + // Sort lastModified in descending order. More recently modified at the + // beginning of the array. + infos.sort((a, b) => b.lastModified - a.lastModified); + + const totalSize = infos.reduce((carry, info) => carry + info.size, 0); + const oldInfos = infos.filter( + info => info.lastModified < Date.now() - this.options.maxAge, + ); + const oldTotalSize = oldInfos.reduce( + (carry, info) => carry + info.size, + 0, + ); + + if (oldInfos.length > 0 && totalSize > this.options.sizeThreshold) { + const newInfos = infos.filter( + info => info.lastModified >= Date.now() - this.options.maxAge, + ); + + for (const info of oldInfos) { + rimraf(join(this.cacheRoot, info.id)); + } + + const newTotalSize = newInfos.reduce( + (carry, info) => carry + info.size, + 0, + ); + + logMessages.deleteOldCaches(compiler, { + infos, + totalSize, + newInfos, + newTotalSize, + oldInfos, + oldTotalSize, + }); + } else { + logMessages.keepCaches(compiler, { + infos, + totalSize, + }); + } + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } finally { + if (typeof resolveLock === 'function') { + deleteLock = null; + resolveLock(); + } + } + }; + + compilerHooks.watchRun.tapPromise( + 'HardSource - PruneCachesSystem', + deleteOldCaches, + ); + compilerHooks.run.tapPromise( + 'HardSource - PruneCachesSystem', + deleteOldCaches, + ); + } +} + +module.exports = PruneCachesSystem; diff --git a/lib/util/log-messages.js b/lib/util/log-messages.js index ed049d6c..8341e7d1 100644 --- a/lib/util/log-messages.js +++ b/lib/util/log-messages.js @@ -91,6 +91,35 @@ exports.configHashBuildWith = (compiler, { cacheDirPath, configHash }) => { ); }; +exports.deleteOldCaches = (compiler, { newTotalSize, oldTotalSize }) => { + const loggerCore = logCore(compiler); + const sizeMB = Math.ceil(newTotalSize / 1024 / 1024); + const deletedSizeMB = Math.ceil(oldTotalSize / 1024 / 1024); + loggerCore.log( + { + id: 'caches--delete-old', + size: newTotalSize, + sizeMB, + deletedSize: oldTotalSize, + deletedSizeMB, + }, + `HardSourceWebpackPlugin is using ${sizeMB} MB of disk space after deleting ${deletedSizeMB} MB.`, + ); +}; + +exports.keepCaches = (compiler, { totalSize }) => { + const loggerCore = logCore(compiler); + const sizeMB = Math.ceil(totalSize / 1024 / 1024); + loggerCore.log( + { + id: 'caches--keep', + size: totalSize, + sizeMB, + }, + `HardSourceWebpackPlugin is using ${sizeMB} MB of disk space.`, + ); +}; + exports.environmentInputs = (compiler, { inputs }) => { const loggerCore = logCore(compiler); loggerCore.log( diff --git a/tests/fixtures/hard-source-prune/config-hash b/tests/fixtures/hard-source-prune/config-hash new file mode 100644 index 00000000..63d8dbd4 --- /dev/null +++ b/tests/fixtures/hard-source-prune/config-hash @@ -0,0 +1 @@ +b \ No newline at end of file diff --git a/tests/fixtures/hard-source-prune/fib/index.js b/tests/fixtures/hard-source-prune/fib/index.js new file mode 100644 index 00000000..bf0205d4 --- /dev/null +++ b/tests/fixtures/hard-source-prune/fib/index.js @@ -0,0 +1,3 @@ +module.exports = function(n) { + return n + (n > 0 ? n - 2 : 0); +}; \ No newline at end of file diff --git a/tests/fixtures/hard-source-prune/index.js b/tests/fixtures/hard-source-prune/index.js new file mode 100644 index 00000000..8acf0395 --- /dev/null +++ b/tests/fixtures/hard-source-prune/index.js @@ -0,0 +1,3 @@ +var fib = require('./fib'); + +console.log(fib(3)); diff --git a/tests/fixtures/hard-source-prune/webpack.config.js b/tests/fixtures/hard-source-prune/webpack.config.js new file mode 100644 index 00000000..f43cce31 --- /dev/null +++ b/tests/fixtures/hard-source-prune/webpack.config.js @@ -0,0 +1,27 @@ +var fs = require('fs'); + +var HardSourceWebpackPlugin = require('../../..'); + +module.exports = { + context: __dirname, + entry: './index.js', + output: { + path: __dirname + '/tmp', + filename: 'main.js', + }, + plugins: [ + new HardSourceWebpackPlugin({ + cacheDirectory: 'cache/[confighash]', + configHash: function(config) { + return fs.readFileSync(__dirname + '/config-hash', 'utf8'); + }, + environmentHash: { + root: __dirname + '/../../..', + }, + cachePrune: { + maxAge: -2000, + sizeThreshold: 0, + }, + }), + ], +}; diff --git a/tests/hard-source.js b/tests/hard-source.js index bbe9f07e..5e7dd21c 100644 --- a/tests/hard-source.js +++ b/tests/hard-source.js @@ -278,4 +278,13 @@ describe('hard-source features', function() { itCompilesTwice('hard-source-exclude-plugin'); itCompilesHardModules('hard-source-exclude-plugin', ['./index.js', '!./fib.js']); + itCompilesChange('hard-source-prune', { + 'config-hash': 'a', + }, { + 'config-hash': 'b', + }, function(output) { + expect(fs.readdirSync(__dirname + '/fixtures/hard-source-prune/tmp/cache')) + .to.have.length(1); + }); + });