diff --git a/package.json b/package.json index 509deeeb304386..5a87bf60eb8e89 100644 --- a/package.json +++ b/package.json @@ -259,6 +259,7 @@ "@types/d3": "^3.5.41", "@types/dedent": "^0.7.0", "@types/del": "^3.0.1", + "@types/delete-empty": "^2.0.0", "@types/elasticsearch": "^5.0.26", "@types/enzyme": "^3.1.12", "@types/eslint": "^4.16.2", @@ -266,6 +267,7 @@ "@types/fetch-mock": "^5.12.2", "@types/getopts": "^2.0.0", "@types/glob": "^5.0.35", + "@types/globby": "^8.0.0", "@types/graphql": "^0.13.1", "@types/hapi": "^17.0.18", "@types/has-ansi": "^3.0.0", @@ -312,6 +314,7 @@ "chromedriver": "2.42.1", "classnames": "2.2.5", "dedent": "^0.7.0", + "delete-empty": "^2.0.0", "enzyme": "3.2.0", "enzyme-adapter-react-16": "^1.1.1", "enzyme-to-json": "3.3.1", diff --git a/src/optimize/watch/optmzr_role.js b/src/optimize/watch/optmzr_role.js index e96e98a339f3f9..6d5df5b6350869 100644 --- a/src/optimize/watch/optmzr_role.js +++ b/src/optimize/watch/optmzr_role.js @@ -17,17 +17,27 @@ * under the License. */ +import { resolve } from 'path'; + import WatchServer from './watch_server'; import WatchOptimizer, { STATUS } from './watch_optimizer'; +import { WatchCache } from './watch_cache'; export default async (kbnServer, kibanaHapiServer, config) => { + const log = (tags, data) => kibanaHapiServer.log(tags, data); + const watchOptimizer = new WatchOptimizer({ - log: (tags, data) => kibanaHapiServer.log(tags, data), + log, uiBundles: kbnServer.uiBundles, profile: config.get('optimize.profile'), sourceMaps: config.get('optimize.sourceMaps'), prebuild: config.get('optimize.watchPrebuild'), unsafeCache: config.get('optimize.unsafeCache'), + watchCache: new WatchCache({ + log, + outputPath: config.get('path.data'), + cachePath: resolve(kbnServer.uiBundles.getCacheDirectory(), '../'), + }) }); const server = new WatchServer( diff --git a/src/optimize/watch/watch_cache.ts b/src/optimize/watch/watch_cache.ts new file mode 100644 index 00000000000000..1cde3498d416dc --- /dev/null +++ b/src/optimize/watch/watch_cache.ts @@ -0,0 +1,157 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'crypto'; +import { readFile, writeFile } from 'fs'; +import { resolve } from 'path'; +import { promisify } from 'util'; + +import del from 'del'; +import deleteEmpty from 'delete-empty'; +import globby from 'globby'; + +const readAsync = promisify(readFile); +const writeAsync = promisify(writeFile); + +interface Params { + log: (tags: string[], data: string) => void; + outputPath: string; + cachePath: string; +} + +interface WatchCacheStateContent { + optimizerConfigSha?: string; + yarnLockSha?: string; +} + +export class WatchCache { + private readonly log: Params['log']; + private readonly outputPath: Params['outputPath']; + private readonly cachePath: Params['cachePath']; + private readonly cacheState: WatchCacheStateContent; + private statePath: string; + private diskCacheState: WatchCacheStateContent; + private isInitialized: boolean; + + constructor(params: Params) { + this.log = params.log; + this.outputPath = params.outputPath; + this.cachePath = params.cachePath; + + this.isInitialized = false; + this.statePath = ''; + this.cacheState = {}; + this.diskCacheState = {}; + this.cacheState.yarnLockSha = ''; + this.cacheState.optimizerConfigSha = ''; + } + + public async tryInit() { + if (!this.isInitialized) { + this.statePath = resolve(this.outputPath, 'watch_optimizer_cache_state.json'); + this.diskCacheState = await this.read(); + this.cacheState.yarnLockSha = await this.buildYarnLockSha(); + this.cacheState.optimizerConfigSha = await this.buildOptimizerConfigSha(); + this.isInitialized = true; + } + } + + public async tryReset() { + await this.tryInit(); + + if (!this.isResetNeeded()) { + return; + } + + await this.reset(); + } + + public async reset() { + this.log(['info', 'optimize:watch_cache'], 'The optimizer watch cache will reset'); + + // start by deleting the state file to lower the + // amount of time that another process might be able to + // successfully read it once we decide to delete it + await del(this.statePath); + + // delete everything in optimize/.cache directory + // except ts-node + await del(await globby([this.cachePath, `!${this.cachePath}/ts-node/**`], { dot: true })); + + // delete some empty folder that could be left + // from the previous cache path reset action + await deleteEmpty(this.cachePath); + + // re-write new cache state file + await this.write(); + + this.log(['info', 'optimize:watch_cache'], 'The optimizer watch cache has reset'); + } + + private async buildShaWithMultipleFiles(filePaths: string[]) { + const shaHash = createHash('sha1'); + + for (const filePath of filePaths) { + try { + shaHash.update(await readAsync(filePath), 'utf8'); + } catch (e) { + /* no-op */ + } + } + + return shaHash.digest('hex'); + } + + private async buildYarnLockSha() { + const kibanaYarnLock = resolve(__dirname, '../../../yarn.lock'); + + return await this.buildShaWithMultipleFiles([kibanaYarnLock]); + } + + private async buildOptimizerConfigSha() { + const baseOptimizer = resolve(__dirname, '../base_optimizer.js'); + + return await this.buildShaWithMultipleFiles([baseOptimizer]); + } + + private isResetNeeded() { + return this.hasYarnLockChanged() || this.hasOptimizerConfigChanged(); + } + + private hasYarnLockChanged() { + return this.cacheState.yarnLockSha !== this.diskCacheState.yarnLockSha; + } + + private hasOptimizerConfigChanged() { + return this.cacheState.optimizerConfigSha !== this.diskCacheState.optimizerConfigSha; + } + + private async write() { + await writeAsync(this.statePath, JSON.stringify(this.cacheState, null, 2), 'utf8'); + this.diskCacheState = this.cacheState; + } + + private async read(): Promise { + try { + return JSON.parse(await readAsync(this.statePath, 'utf8')); + } catch (error) { + return {}; + } + } +} diff --git a/src/optimize/watch/watch_optimizer.js b/src/optimize/watch/watch_optimizer.js index 7c033d50bf922d..a4932e4247910b 100644 --- a/src/optimize/watch/watch_optimizer.js +++ b/src/optimize/watch/watch_optimizer.js @@ -36,6 +36,7 @@ export default class WatchOptimizer extends BaseOptimizer { super(opts); this.log = opts.log || (() => null); this.prebuild = opts.prebuild || false; + this.watchCache = opts.watchCache; this.status$ = new Rx.ReplaySubject(1); } @@ -43,6 +44,9 @@ export default class WatchOptimizer extends BaseOptimizer { this.initializing = true; this.initialBuildComplete = false; + // try reset the watch optimizer cache + await this.watchCache.tryReset(); + // log status changes this.status$.subscribe(this.onStatusChangeHandler); await this.uiBundles.resetBundleDir(); diff --git a/src/server/logging/log_format_string.js b/src/server/logging/log_format_string.js index db438840a222f6..a55e26a7c3c6c8 100644 --- a/src/server/logging/log_format_string.js +++ b/src/server/logging/log_format_string.js @@ -49,6 +49,7 @@ const typeColors = { optmzr: 'white', manager: 'green', optimize: 'magentaBright', + 'optimize:watch_cache': 'magentaBright', listening: 'magentaBright', scss: 'magentaBright', }; diff --git a/yarn.lock b/yarn.lock index b921e9e93bccf0..1e789662b77f1e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1095,6 +1095,11 @@ resolved "https://registry.yarnpkg.com/@types/delay/-/delay-2.0.1.tgz#61bcf318a74b61e79d1658fbf054f984c90ef901" integrity sha512-D1/YuYOcdOIdaQnaiUJ77VcilVvESkynw79CtGqpjkXyv4OUezEVZtdXnSOwXL8Zcelu66QbyC8QQcVQ/ZPdig== +"@types/delete-empty@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@types/delete-empty/-/delete-empty-2.0.0.tgz#1647ae9e68f708a6ba778531af667ec55bc61964" + integrity sha512-sq+kwx8zA9BSugT9N+Jr8/uWjbHMZ+N/meJEzRyT3gmLq/WMtx/iSIpvdpmBUi/cvXl6Kzpvve8G2ESkabFwmg== + "@types/elasticsearch@^5.0.26": version "5.0.28" resolved "https://registry.yarnpkg.com/@types/elasticsearch/-/elasticsearch-5.0.28.tgz#0e4cdf7d9c9a3fe901c0da4fb9ad824c6d3b4091" @@ -1178,6 +1183,14 @@ dependencies: "@types/glob" "*" +"@types/globby@^8.0.0": + version "8.0.0" + resolved "https://registry.yarnpkg.com/@types/globby/-/globby-8.0.0.tgz#7bd10eaf802e1e11afdb1e5436cf472ddf4c0dd2" + integrity sha512-xDtsX5tlctxJzvg29r/LN12z30oJpoFP9cE8eJ8nY5cbSvN0c0RdRHrVlEq4LRh362Sd+JsqxJ3QWw0Wnyto8w== + dependencies: + "@types/glob" "*" + fast-glob "^2.0.2" + "@types/got@^7.1.7": version "7.1.8" resolved "https://registry.yarnpkg.com/@types/got/-/got-7.1.8.tgz#c5f421b25770689bf8948b1241f710d71a00d7dd" @@ -2022,6 +2035,13 @@ ansi-gray@^0.1.1: dependencies: ansi-wrap "0.1.0" +ansi-green@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/ansi-green/-/ansi-green-0.1.1.tgz#8a5d9a979e458d57c40e33580b37390b8e10d0f7" + integrity sha1-il2al55FjVfEDjNYCzc5C44Q0Pc= + dependencies: + ansi-wrap "0.1.0" + ansi-html@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e" @@ -6714,6 +6734,15 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= +delete-empty@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/delete-empty/-/delete-empty-2.0.0.tgz#dcf7c4f93a98445119acd57b137d13e7af78fa39" + integrity sha512-voZ8OiMkVR9MOTTHZ5P0DaMDtIW6xEbXZeADp6U8uwxIJFhs2hRwyIlUZIs5hR4YIp9VYBURqZrV6Yz0ozhVpg== + dependencies: + log-ok "^0.1.1" + relative "^3.0.2" + rimraf "^2.6.2" + depd@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" @@ -13623,6 +13652,14 @@ lodash@~4.3.0: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.3.0.tgz#efd9c4a6ec53f3b05412429915c3e4824e4d25a4" integrity sha1-79nEpuxT87BUEkKZFcPkgk5NJaQ= +log-ok@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/log-ok/-/log-ok-0.1.1.tgz#bea3dd36acd0b8a7240d78736b5b97c65444a334" + integrity sha1-vqPdNqzQuKckDXhza1uXxlREozQ= + dependencies: + ansi-green "^0.1.1" + success-symbol "^0.1.0" + log-symbols@^1.0.1, log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" @@ -18025,6 +18062,13 @@ relateurl@0.2.x: resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9" integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk= +relative@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/relative/-/relative-3.0.2.tgz#0dcd8ec54a5d35a3c15e104503d65375b5a5367f" + integrity sha1-Dc2OxUpdNaPBXhBFA9ZTdbWlNn8= + dependencies: + isobject "^2.0.0" + remark-parse@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-5.0.0.tgz#4c077f9e499044d1d5c13f80d7a98cf7b9285d95" @@ -19982,6 +20026,11 @@ subtext@6.x.x: pez "4.x.x" wreck "14.x.x" +success-symbol@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/success-symbol/-/success-symbol-0.1.0.tgz#24022e486f3bf1cdca094283b769c472d3b72897" + integrity sha1-JAIuSG878c3KCUKDt2nEctO3KJc= + sudo-block@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/sudo-block/-/sudo-block-1.2.0.tgz#cc539bf8191624d4f507d83eeb45b4cea27f3463"