Add jenkins:report task for test failures

package.json

@@ -210,6 +210,7 @@
     "@kbn/eslint-plugin-license-header": "link:packages/kbn-eslint-plugin-license-header",
     "@kbn/plugin-generator": "link:packages/kbn-plugin-generator",
     "@kbn/test": "link:packages/kbn-test",
+    "@octokit/rest": "^15.10.0",
     "@types/angular": "^1.6.45",
     "@types/babel-core": "^6.25.5",
     "@types/bluebird": "^3.1.1",

src/dev/failed_tests/report.js

@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import xml2js from 'xml2js';
+import vfs from 'vinyl-fs';
+import es from 'event-stream';
+import { getGithubClient, markdownMetadata, paginate } from '../github_utils';
+import { find } from 'lodash';
+import stripAnsi from 'strip-ansi';
+
+const GITHUB_FLAKY_TEST_LABEL = 'flaky-failing-test';
+const GITHUB_OWNER = 'elastic';
+const GITHUB_REPO = 'kibana';
+const BUILD_URL = process.env.BUILD_URL;
+
+/**
+ * Parses junit XML files into JSON
+ */
+const mapXml = es.map((file, cb) => {
+  xml2js.parseString(file.contents.toString(), (err, result) => {
+    cb(null, result);
+  });
+});
+
+/**
+ * Filters all testsuites to find failed testcases
+ */
+const filterFailures = es.map((testSuite, cb) => {
+  const testFiles = testSuite.testsuites.testsuite;
+
+  const failures = testFiles.reduce((failures, testFile) => {
+    for (const testCase of testFile.testcase) {
+      if (testCase.failure) {
+        // unwrap xml weirdness
+        failures.push({
+          ...testCase.$,
+          // Strip ANSI color characters
+          failure: stripAnsi(testCase.failure[0])
+        });
+      }
+    }
+
+    return failures;
+  }, []);
+
+  console.log(`Found ${failures.length} test failures`);
+
+  cb(null, failures);
+});
+
+/**
+ * Creates and updates github issues for the given testcase failures.
+ */
+const updateGithubIssues = (githubClient, issues) => {
+  return es.map(async (failureCases, cb) => {
+
+    const issueOps = failureCases.map(async (failureCase) => {
+      const existingIssue = find(issues, (issue) => {
+        return markdownMetadata.get(issue.body, 'test.class') === failureCase.classname &&
+          markdownMetadata.get(issue.body, 'test.name') === failureCase.name;
+      });
+
+      if (existingIssue) {
+        // Increment failCount
+        const newCount = (markdownMetadata.get(existingIssue.body, 'test.failCount') || 0) + 1;
+        const newBody = markdownMetadata.set(existingIssue.body, 'test.failCount', newCount);
+
+        await githubClient.issues.edit({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          number: existingIssue.number,
+          state: 'open',  // Reopen issue if it was closed.
+          body: newBody
+        });
+
+        // Append a new comment
+        await githubClient.issues.createComment({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          number: existingIssue.number,
+          body: `New failure: [Jenkins Build](${BUILD_URL})`
+        });
+
+        console.log(`Updated issue ${existingIssue.html_url}, failCount: ${newCount}`);
+      } else {
+        let body = 'A test failed on a tracked branch\n' +
+          '```\n' + failureCase.failure + '\n```\n' +
+          `First failure: [Jenkins Build](${BUILD_URL})`;
+        body = markdownMetadata.set(body, {
+          'test.class': failureCase.classname,
+          'test.name': failureCase.name,
+          'test.failCount': 1
+        });
+
+        const newIssue = await githubClient.issues.create({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          title: `Failing test: ${failureCase.classname} - ${failureCase.name}`,
+          body: body,
+          labels: [GITHUB_FLAKY_TEST_LABEL]
+        });
+
+        console.log(`Created issue ${newIssue.data.html_url}`);
+      }
+    });
+
+    Promise
+      .all(issueOps)
+      .then(() => cb(null, failureCases))
+      .catch(e => cb(e));
+  });
+};
+
+/**
+ * Scans all junit XML files in ./target/junit/ and reports any found test failures to Github Issues.
+ */
+export async function reportFailedTests(done) {
+  const githubClient = getGithubClient();
+  const issues = await paginate(githubClient, githubClient.issues.getForRepo({
+    owner: GITHUB_OWNER,
+    repo: GITHUB_REPO,
+    labels: GITHUB_FLAKY_TEST_LABEL,
+    state: 'all',
+    per_page: 100
+  }));
+
+  vfs
+    .src(['./target/junit/**/*.xml'])
+    .pipe(mapXml)
+    .pipe(filterFailures)
+    .pipe(updateGithubIssues(githubClient, issues))
+    .on('done', done);
+}

src/dev/github_utils/index.js

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Octokit from '@octokit/rest';
+import { markdownMetadata } from './metadata';
+
+export { markdownMetadata };
+
+export function getGithubClient() {
+  const client = new Octokit();
+  client.authenticate({
+    type: 'token',
+    token: process.env.GITHUB_TOKEN
+  });
+
+  return client;
+}
+
+export async function paginate(client, promise) {
+  let response = await promise;
+  let { data } = response;
+  while (client.hasNextPage(response)) {
+    response = await client.getNextPage(response);
+    data = data.concat(response.data);
+  }
+  return data;
+}

src/dev/github_utils/metadata.js

@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const REGEX = /\n\n<!-- kibanaCiData = (.*) -->/;
+
+/**
+ * Allows retrieving and setting key/value pairs on a Github Issue. Keys and values must be JSON-serializable.
+ * Borrowed heavily from https://github.com/probot/metadata/blob/6ae1523d5035ba727d09c0e7f77a6a154d9a4777/index.js
+ *
+ * `body` is a string that contains markdown and any existing metadata (eg. an issue or comment body)
+ * `prefix` is a string that can be used to namespace the metadata, defaults to `ci`.
+ */
+export const markdownMetadata = {
+  get(body, key = null, prefix = 'failed-test') {
+    const match = body.match(REGEX);
+
+    if (match) {
+      const data = JSON.parse(match[1])[prefix];
+      return key ? data && data[key] : data;
+    } else {
+      return null;
+    }
+  },
+
+  /**
+   * Set data on the body. Can either be set individually with `key` and `value` OR
+   */
+  set(body, key, value, prefix = 'failed-test') {
+    let newData = {};
+    // If second arg is an object, use all supplied values.
+    if (typeof key === 'object') {
+      newData = key;
+      prefix = value || prefix;  // need to move third arg to prefix.
+    } else {
+      newData[key] = value;
+    }
+
+    let data = {};
+
+    body = body.replace(REGEX, (_, json) => {
+      data = JSON.parse(json);
+      return '';
+    });
+
+    if (!data[prefix]) data[prefix] = {};
+
+    Object.assign(data[prefix], newData);
+
+    return `${body}\n\n<!-- kibanaCiData = ${JSON.stringify(data)} -->`;
+  }
+};

tasks/jenkins.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { reportFailedTests } from '../src/dev/failed_tests/report';
+
 module.exports = function (grunt) {
   grunt.registerTask('jenkins:docs', [
     'docker:docs'
@@ -44,4 +46,12 @@ module.exports = function (grunt) {
     'run:functionalTestsRelease',
     'run:pluginFunctionalTestsRelease',
   ]);
+
+  grunt.registerTask(
+    'jenkins:report',
+    'Reports failed tests found in junit xml files to Github issues',
+    function () {
+      reportFailedTests(this.async());
+    }
+  );
 };

test/scripts/jenkins_report_failed_tests.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+xvfb-run "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:report;