Watch optimizer cache invalidation

package.json

@@ -231,13 +231,16 @@
     "@types/classnames": "^2.2.3",
     "@types/d3": "^5.0.0",
     "@types/dedent": "^0.7.0",
+    "@types/del": "^3.0.1",
+    "@types/delete-empty": "^2.0.0",
     "@types/elasticsearch": "^5.0.26",
     "@types/enzyme": "^3.1.12",
     "@types/eslint": "^4.16.2",
     "@types/execa": "^0.9.0",
     "@types/fetch-mock": "^5.12.2",
     "@types/getopts": "^2.0.0",
     "@types/glob": "^5.0.35",
+    "@types/globby": "^8.0.0",
     "@types/hapi-latest": "npm:@types/[email protected]",
     "@types/has-ansi": "^3.0.0",
     "@types/jest": "^23.3.1",
@@ -247,6 +250,7 @@
     "@types/listr": "^0.13.0",
     "@types/lodash": "^3.10.1",
     "@types/minimatch": "^2.0.29",
+    "@types/mkdirp": "^0.5.2",
     "@types/moment-timezone": "^0.5.8",
     "@types/mustache": "^0.8.31",
     "@types/node": "^8.10.20",
@@ -277,6 +281,7 @@
     "chromedriver": "2.42.1",
     "classnames": "2.2.5",
     "dedent": "^0.7.0",
+    "delete-empty": "^2.0.0",
     "enzyme": "3.2.0",
     "enzyme-adapter-react-16": "^1.1.1",
     "enzyme-to-json": "3.3.1",

src/optimize/watch/optmzr_role.js

@@ -17,21 +17,33 @@
  * under the License.
  */
 
+import { resolve } from 'path';
+
 import WatchServer from './watch_server';
 import WatchOptimizer from './watch_optimizer';
+// import { DllCompiler } from '../dynamic_dll_plugin';
+import { fromRoot } from '../../utils'; // TODO: remove when merge with master
+import { WatchCache } from './watch_cache';
 
 export default async (kbnServer, kibanaHapiServer, config) => {
+  const log = (tags, data) => kibanaHapiServer.log(tags, data);
   const server = new WatchServer(
     config.get('optimize.watchHost'),
     config.get('optimize.watchPort'),
     config.get('server.basePath'),
     new WatchOptimizer({
-      log: (tags, data) => kibanaHapiServer.log(tags, data),
+      log,
       uiBundles: kbnServer.uiBundles,
       profile: config.get('optimize.profile'),
       sourceMaps: config.get('optimize.sourceMaps'),
       prebuild: config.get('optimize.watchPrebuild'),
       unsafeCache: config.get('optimize.unsafeCache'),
+      watchCache: new WatchCache({
+        log,
+        outputPath: config.get('path.data'),
+        dllsPath: fromRoot('./dlls'), // TODO: replace by DllCompiler.getRawDllConfig().outputPath when merge with master
+        cachePath: resolve(kbnServer.uiBundles.getCacheDirectory(), '../'),
+      })
     })
   );
 

src/optimize/watch/watch_cache.ts

@@ -0,0 +1,173 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createHash } from 'crypto';
+import { readFile, writeFile } from 'fs';
+import { resolve } from 'path';
+import { promisify } from 'util';
+
+import del from 'del';
+import deleteEmpty from 'delete-empty';
+import globby from 'globby';
+import mkdirp from 'mkdirp';
+
+const mkdirpAsync = promisify(mkdirp);
+const readAsync = promisify(readFile);
+const writeAsync = promisify(writeFile);
+
+interface Params {
+  log: (tags: string[], data: string) => void;
+  outputPath: string;
+  dllsPath: string;
+  cachePath: string;
+}
+
+interface WatchCacheStateContent {
+  optimizerConfigSha?: string;
+  yarnLockSha?: string;
+}
+
+export class WatchCache {
+  private readonly log: Params['log'];
+  private readonly outputPath: Params['outputPath'];
+  private readonly dllsPath: Params['dllsPath'];
+  private readonly cachePath: Params['cachePath'];
+  private readonly cacheState: WatchCacheStateContent;
+  private statePath: string;
+  private diskCacheState: WatchCacheStateContent;
+  private isInitialized: boolean;
+
+  constructor(params: Params) {
+    this.log = params.log;
+    this.outputPath = params.outputPath;
+    this.dllsPath = params.dllsPath;
+    this.cachePath = params.cachePath;
+
+    this.isInitialized = false;
+    this.statePath = '';
+    this.cacheState = {};
+    this.diskCacheState = {};
+    this.cacheState.yarnLockSha = '';
+    this.cacheState.optimizerConfigSha = '';
+  }
+
+  public async tryInit() {
+    if (!this.isInitialized) {
+      this.statePath = resolve(this.outputPath, 'watch_optimizer_cache_state.json');
+      this.diskCacheState = await this.read();
+      this.cacheState.yarnLockSha = await this.buildYarnLockSha();
+      this.cacheState.optimizerConfigSha = await this.buildOptimizerConfigSha();
+      this.isInitialized = true;
+    }
+  }
+
+  public async tryReset() {
+    await this.tryInit();
+
+    if (!this.isResetNeeded()) {
+      return;
+    }
+
+    await this.reset();
+  }
+
+  public async reset() {
+    this.log(['info', 'optimize:watch_cache'], 'The optimizer watch cache will reset');
+
+    // start by deleting the state file to lower the
+    // amount of time that another process might be able to
+    // successfully read it once we decide to delete it
+    await del(this.statePath);
+
+    // first delete some empty folder that left
+    // from any previous cache reset action
+    await deleteEmpty(`${this.cachePath}`);
+
+    // delete everything in optimize/.cache directory
+    // except ts-node
+    await del(await globby([`${this.cachePath}`, `!${this.cachePath}/ts-node/**`], { dot: true }));
+
+    // delete dlls
+    await del(this.dllsPath);
+    await mkdirpAsync(this.dllsPath);
+
+    // re-write new cache state file
+    await this.write();
+
+    this.log(['info', 'optimize:watch_cache'], 'The optimizer watch cache has reset');
+  }
+
+  private async buildShaWithMultipleFiles(filePaths: string[]) {
+    const shaHash = createHash('sha1');
+
+    for (const filePath of filePaths) {
+      try {
+        shaHash.update(await readAsync(filePath), 'utf8');
+      } catch (e) {
+        /* no-op */
+      }
+    }
+
+    return shaHash.digest('hex');
+  }
+
+  private async buildYarnLockSha() {
+    const kibanaYarnLock = resolve(__dirname, '../../../yarn.lock');
+    const xpackYarnLock = resolve(__dirname, '../../../x-pack/yarn.lock');
+
+    return await this.buildShaWithMultipleFiles([kibanaYarnLock, xpackYarnLock]);
+  }
+
+  private async buildOptimizerConfigSha() {
+    const baseOptimizer = resolve(__dirname, '../base_optimizer.js');
+    const dynamicDllConfigModel = resolve(__dirname, '../dynamic_dll_plugin/dll_config_model.js');
+    const dynamicDllPlugin = resolve(__dirname, '../dynamic_dll_plugin/dynamic_dll_plugin.js');
+
+    return await this.buildShaWithMultipleFiles([
+      baseOptimizer,
+      dynamicDllConfigModel,
+      dynamicDllPlugin,
+    ]);
+  }
+
+  private isResetNeeded() {
+    return this.hasYarnLockChanged() || this.hasOptimizerConfigChanged();
+  }
+
+  private hasYarnLockChanged() {
+    return this.cacheState.yarnLockSha !== this.diskCacheState.yarnLockSha;
+  }
+
+  private hasOptimizerConfigChanged() {
+    return this.cacheState.optimizerConfigSha !== this.diskCacheState.optimizerConfigSha;
+  }
+
+  private async write() {
+    await writeAsync(this.statePath, JSON.stringify(this.cacheState, null, 2), 'utf8');
+    this.diskCacheState = this.cacheState;
+  }
+
+  private async read(): Promise<WatchCacheStateContent> {
+    try {
+      return JSON.parse(await readAsync(this.statePath, 'utf8'));
+    } catch (error) {
+      return {};
+    }
+  }
+}

src/optimize/watch/watch_optimizer.js

@@ -36,13 +36,17 @@ export default class WatchOptimizer extends BaseOptimizer {
     super(opts);
     this.log = opts.log || (() => null);
     this.prebuild = opts.prebuild || false;
+    this.watchCache = opts.watchCache;
     this.status$ = new Rx.ReplaySubject(1);
   }
 
   async init() {
     this.initializing = true;
     this.initialBuildComplete = false;
 
+    // try reset the watch optimizer cache
+    await this.watchCache.tryReset();
+
     // log status changes
     this.status$.subscribe(this.onStatusChangeHandler);
     await this.uiBundles.resetBundleDir();

src/server/logging/log_format_string.js

@@ -49,6 +49,7 @@ const typeColors = {
   optmzr: 'white',
   manager: 'green',
   optimize: 'magentaBright',
+  'optimize:watch_cache': 'magentaBright',
   listening: 'magentaBright',
   scss: 'magentaBright',
 };