Add telemetry

.vscode/launch.json

@@ -28,7 +28,7 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--disable-extensions",
+        "--disable-extension=ms-kubernetes-tools.vscode-kubernetes-tools",
         "--extensionDevelopmentPath=${workspaceFolder}",
         "--extensionTestsPath=${workspaceFolder}/out/test",
         "${workspaceRoot}/test/testFixture"

README.md

@@ -232,6 +232,10 @@ e.g.
 }
 ```
 
+## Data and Telemetry
+
+The `vscode-yaml` extension collects anonymous [usage data](USAGE_DATA.md) and sends it to Red Hat servers to help improve our products and services. Read our [privacy statement](https://developers.redhat.com/article/tool-data-collection) to learn more. This extension respects the `redhat.telemetry.enabled` setting which you can learn more about at https://github.com/redhat-developer/vscode-commons#how-to-disable-telemetry-reporting
+
 ## How to contribute
 
 The instructions are available in the [contribution guide](CONTRIBUTING.md).

USAGE_DATA.md

@@ -0,0 +1,20 @@
+# Data collection
+
+vscode-yaml has opt-in telemetry collection, provided by [vscode-commons](https://github.com/redhat-developer/vscode-commons).
+
+## What's included in the vscode-yaml telemetry data
+
+- yaml-language-server start
+- errors during yaml language server start
+- any errors from LSP requests
+- `kubernetes` schema usage
+
+## What's included in the general telemetry data
+
+Please see the
+[vscode-commons data collection information](https://github.com/redhat-developer/vscode-commons/blob/master/USAGE_DATA.md#other-extensions)
+for information on what data it collects.
+
+## How to opt in or out
+
+Use the `redhat.telemetry.enabled` setting in order to enable or disable telemetry collection.

package.json

@@ -164,6 +164,9 @@
       }
     }
   },
+  "extensionDependencies": [
+    "redhat.vscode-commons"
+  ],
   "scripts": {
     "build": "yarn run clean && yarn run lint && yarn run vscode:prepublish",
     "check-dependencies": "node ./scripts/check-dependencies.js",
@@ -175,24 +178,31 @@
     "vscode:prepublish": "tsc -p ./"
   },
   "devDependencies": {
+    "@types/chai": "^4.2.12",
     "@types/fs-extra": "^9.0.6",
     "@types/mocha": "^2.2.48",
     "@types/node": "^12.12.6",
+    "@types/sinon": "^9.0.5",
+    "@types/sinon-chai": "^3.2.5",
     "@types/vscode": "^1.52.0",
     "@typescript-eslint/eslint-plugin": "^4.16.1",
     "@typescript-eslint/parser": "^4.16.1",
+    "chai": "^4.2.0",
     "eslint": "^7.6.0",
     "eslint-config-prettier": "^6.11.0",
     "eslint-plugin-prettier": "^3.1.4",
     "glob": "^7.1.6",
     "mocha": "^8.0.1",
     "prettier": "^2.0.5",
     "rimraf": "^3.0.2",
+    "sinon": "^9.0.3",
+    "sinon-chai": "^3.5.0",
     "ts-node": "^3.3.0",
     "typescript": "4.1.2",
     "vscode-test": "^1.4.0"
   },
   "dependencies": {
+    "@redhat-developer/vscode-redhat-telemetry": "0.0.18",
     "fs-extra": "^9.1.0",
     "request-light": "^0.4.0",
     "vscode-languageclient": "7.0.0",

src/extension.ts

@@ -23,6 +23,8 @@ import { joinPath } from './paths';
 import { getJsonSchemaContent, JSONSchemaDocumentContentProvider } from './json-schema-content-provider';
 import { JSONSchemaCache } from './json-schema-cache';
 import { getConflictingExtensions, showUninstallConflictsNotification } from './extensionConflicts';
+import { getTelemetryService } from '@redhat-developer/vscode-redhat-telemetry';
+import { TelemetryErrorHandler, TelemetryOutputChannel } from './telemetry';
 
 export interface ISchemaAssociations {
   [pattern: string]: string[];
@@ -77,7 +79,13 @@ namespace ResultLimitReachedNotification {
 
 let client: LanguageClient;
 
-export function activate(context: ExtensionContext): SchemaExtensionAPI {
+const lsName = 'YAML Support';
+
+export async function activate(context: ExtensionContext): Promise<SchemaExtensionAPI> {
+  // Create Telemetry Service
+  const telemetry = await getTelemetryService('redhat.vscode-yaml');
+  telemetry.sendStartupEvent();
+
   // The YAML language server is implemented in node
   const serverModule = context.asAbsolutePath(
     path.join('node_modules', 'yaml-language-server', 'out', 'server', 'src', 'server.js')
@@ -93,6 +101,8 @@ export function activate(context: ExtensionContext): SchemaExtensionAPI {
     debug: { module: serverModule, transport: TransportKind.ipc, options: debugOptions },
   };
 
+  const telemetryErrorHandler = new TelemetryErrorHandler(telemetry, lsName, 4);
+  const outputChannel = window.createOutputChannel(lsName);
   // Options to control the language client
   const clientOptions: LanguageClientOptions = {
     // Register the server for on disk and newly created YAML documents
@@ -104,10 +114,12 @@ export function activate(context: ExtensionContext): SchemaExtensionAPI {
       fileEvents: [workspace.createFileSystemWatcher('**/*.?(e)y?(a)ml'), workspace.createFileSystemWatcher('**/*.json')],
     },
     revealOutputChannelOn: RevealOutputChannelOn.Never,
+    errorHandler: telemetryErrorHandler,
+    outputChannel: new TelemetryOutputChannel(outputChannel, telemetry),
   };
 
   // Create the language client and start it
-  client = new LanguageClient('yaml', 'YAML Support', serverOptions, clientOptions);
+  client = new LanguageClient('yaml', lsName, serverOptions, clientOptions);
 
   const schemaCache = new JSONSchemaCache(context.globalStoragePath, context.globalState, client.outputChannel);
   const disposable = client.start();
@@ -124,6 +136,12 @@ export function activate(context: ExtensionContext): SchemaExtensionAPI {
     )
   );
 
+  context.subscriptions.push(
+    client.onTelemetry((e) => {
+      telemetry.send(e);
+    })
+  );
+
   findConflicts();
   client.onReady().then(() => {
     // Send a notification to the server with any YAML schema associations in all extensions
@@ -149,6 +167,7 @@ export function activate(context: ExtensionContext): SchemaExtensionAPI {
       return getJsonSchemaContent(uri, schemaCache);
     });
 
+    telemetry.send({ name: 'yaml.server.initialized' });
     // Adapted from:
     // https://github.com/microsoft/vscode/blob/94c9ea46838a9a619aeafb7e8afd1170c967bb55/extensions/json-language-features/client/src/jsonClient.ts#L305-L318
     client.onNotification(ResultLimitReachedNotification.type, async (message) => {

src/telemetry.ts

@@ -0,0 +1,80 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Red Hat, Inc. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { TelemetryService } from '@redhat-developer/vscode-redhat-telemetry/lib';
+import { CloseAction, ErrorAction, ErrorHandler, Message } from 'vscode-languageclient/node';
+import * as vscode from 'vscode';
+
+export class TelemetryErrorHandler implements ErrorHandler {
+  private restarts: number[] = [];
+  constructor(
+    private readonly telemetry: TelemetryService,
+    private readonly name: string,
+    private readonly maxRestartCount: number
+  ) {}
+
+  error(error: Error, message: Message, count: number): ErrorAction {
+    this.telemetry.send({ name: 'yaml.lsp.error', properties: { jsonrpc: message.jsonrpc, error: error.message } });
+    if (count && count <= 3) {
+      return ErrorAction.Continue;
+    }
+    return ErrorAction.Shutdown;
+  }
+  closed(): CloseAction {
+    this.restarts.push(Date.now());
+    if (this.restarts.length <= this.maxRestartCount) {
+      return CloseAction.Restart;
+    } else {
+      const diff = this.restarts[this.restarts.length - 1] - this.restarts[0];
+      if (diff <= 3 * 60 * 1000) {
+        vscode.window.showErrorMessage(
+          `The ${this.name} server crashed ${
+            this.maxRestartCount + 1
+          } times in the last 3 minutes. The server will not be restarted.`
+        );
+        return CloseAction.DoNotRestart;
+      } else {
+        this.restarts.shift();
+        return CloseAction.Restart;
+      }
+    }
+  }
+}
+
+export class TelemetryOutputChannel implements vscode.OutputChannel {
+  constructor(private readonly delegate: vscode.OutputChannel, private readonly telemetry: TelemetryService) {}
+
+  get name(): string {
+    return this.delegate.name;
+  }
+  append(value: string): void {
+    this.checkError(value);
+    this.delegate.append(value);
+  }
+  appendLine(value: string): void {
+    this.checkError(value);
+    this.delegate.appendLine(value);
+  }
+
+  private checkError(value: string): void {
+    if (value.startsWith('[Error') || value.startsWith('  Message: Request')) {
+      this.telemetry.send({ name: 'yaml.server.error', properties: { error: value } });
+    }
+  }
+  clear(): void {
+    this.delegate.clear();
+  }
+  show(preserveFocus?: boolean): void;
+  show(column?: vscode.ViewColumn, preserveFocus?: boolean): void;
+  show(column?: never, preserveFocus?: boolean): void {
+    this.delegate.show(column, preserveFocus);
+  }
+  hide(): void {
+    this.delegate.hide();
+  }
+  dispose(): void {
+    this.delegate.dispose();
+  }
+}

test/telemetry.test.ts

@@ -0,0 +1,109 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Red Hat, Inc. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+import * as sinon from 'sinon';
+import * as sinonChai from 'sinon-chai';
+import * as chai from 'chai';
+import * as vscode from 'vscode';
+import { TelemetryErrorHandler, TelemetryOutputChannel } from '../src/telemetry';
+import { TelemetryEvent, TelemetryService } from '@redhat-developer/vscode-redhat-telemetry/lib/interfaces/telemetry';
+
+const expect = chai.expect;
+chai.use(sinonChai);
+class TelemetryStub implements TelemetryService {
+  sendStartupEvent(): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  send(event: TelemetryEvent): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+  sendShutdownEvent(): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+  flushQueue(): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+  dispose(): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+}
+describe('Telemetry Test', () => {
+  const sandbox = sinon.createSandbox();
+  const testOutputChannel = vscode.window.createOutputChannel('YAML_TEST');
+  afterEach(() => {
+    sandbox.restore();
+  });
+  describe('TelemetryOutputChannel', () => {
+    let telemetryChannel: TelemetryOutputChannel;
+    let outputChannel: sinon.SinonStubbedInstance<vscode.OutputChannel>;
+    let telemetry: sinon.SinonStubbedInstance<TelemetryService>;
+    beforeEach(() => {
+      outputChannel = sandbox.stub(testOutputChannel);
+      telemetry = sandbox.stub(new TelemetryStub());
+      telemetryChannel = new TelemetryOutputChannel(
+        (outputChannel as unknown) as vscode.OutputChannel,
+        (telemetry as unknown) as TelemetryService
+      );
+    });
+
+    it('should delegate "append" method', () => {
+      telemetryChannel.append('Some');
+      expect(outputChannel.append).calledOnceWith('Some');
+    });
+
+    it('should delegate "appendLine" method', () => {
+      telemetryChannel.appendLine('Some');
+      expect(outputChannel.appendLine).calledOnceWith('Some');
+    });
+
+    it('should delegate "clear" method', () => {
+      telemetryChannel.clear();
+      expect(outputChannel.clear).calledOnce;
+    });
+
+    it('should delegate "dispose" method', () => {
+      telemetryChannel.dispose();
+      expect(outputChannel.dispose).calledOnce;
+    });
+
+    it('should delegate "hide" method', () => {
+      telemetryChannel.hide();
+      expect(outputChannel.hide).calledOnce;
+    });
+
+    it('should delegate "show" method', () => {
+      telemetryChannel.show();
+      expect(outputChannel.show).calledOnce;
+    });
+
+    it('should send telemetry if log error in "append"', () => {
+      telemetryChannel.append('[Error Some');
+      expect(telemetry.send).calledOnceWith({ name: 'yaml.server.error', properties: { error: '[Error Some' } });
+    });
+
+    it('should send telemetry if log error on "appendLine"', () => {
+      telemetryChannel.appendLine('[Error Some');
+      expect(telemetry.send).calledOnceWith({ name: 'yaml.server.error', properties: { error: '[Error Some' } });
+    });
+  });
+
+  describe('TelemetryErrorHandler', () => {
+    let telemetry: sinon.SinonStubbedInstance<TelemetryService>;
+    let errorHandler: TelemetryErrorHandler;
+
+    beforeEach(() => {
+      telemetry = sandbox.stub(new TelemetryStub());
+      errorHandler = new TelemetryErrorHandler(telemetry, 'YAML LS', 3);
+    });
+
+    it('should log telemetry on error', () => {
+      errorHandler.error(new Error('Some'), { jsonrpc: 'Error message' }, 3);
+      expect(telemetry.send).calledOnceWith({
+        name: 'yaml.lsp.error',
+        properties: { jsonrpc: 'Error message', error: 'Some' },
+      });
+    });
+  });
+});

test/testRunner.ts

@@ -3,11 +3,18 @@
  * Licensed under the MIT License. See License.txt in the project root for license information.
  * ------------------------------------------------------------------------------------------ */
 import * as path from 'path';
-
-import { runTests } from 'vscode-test';
+import * as cp from 'child_process';
+import { runTests, downloadAndUnzipVSCode, resolveCliPathFromVSCodeExecutablePath } from 'vscode-test';
 
 async function main(): Promise<void> {
   try {
+    const executable = await downloadAndUnzipVSCode();
+    const cliPath = resolveCliPathFromVSCodeExecutablePath(executable);
+    const dependencies = ['redhat.vscode-commons'];
+    for (const dep of dependencies) {
+      const installLog = cp.execSync(`"${cliPath}" --install-extension ${dep}`);
+      console.log(installLog.toString());
+    }
     // The folder containing the Extension Manifest package.json
     // Passed to `--extensionDevelopmentPath`
     const extensionDevelopmentPath = path.resolve(__dirname, '../../');
@@ -18,9 +25,10 @@ async function main(): Promise<void> {
 
     // Download VS Code, unzip it and run the integration test
     await runTests({
+      vscodeExecutablePath: executable,
       extensionDevelopmentPath,
       extensionTestsPath,
-      launchArgs: ['--disable-extensions', '.'],
+      launchArgs: ['--disable-extension=ms-kubernetes-tools.vscode-kubernetes-tools', '.'],
     });
   } catch (err) {
     console.error('Failed to run tests');