From cf355189a646e50a0b7aa2740e31f57dddbb6ba5 Mon Sep 17 00:00:00 2001 From: Josh Dover Date: Tue, 4 Sep 2018 13:42:22 -0500 Subject: [PATCH] Add jenkins:report task --- package.json | 1 + src/dev/failed_tests/report.js | 149 ++++++++++++++++++++ src/dev/github_utils/index.js | 44 ++++++ src/dev/github_utils/metadata.js | 67 +++++++++ tasks/jenkins.js | 10 ++ test/scripts/jenkins_report_failed_tests.sh | 5 + yarn.lock | 47 +++++- 7 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 src/dev/failed_tests/report.js create mode 100644 src/dev/github_utils/index.js create mode 100644 src/dev/github_utils/metadata.js create mode 100755 test/scripts/jenkins_report_failed_tests.sh diff --git a/package.json b/package.json index 5ea6728b6458ea8..be85f0d70be9d0b 100644 --- a/package.json +++ b/package.json @@ -210,6 +210,7 @@ "@kbn/eslint-plugin-license-header": "link:packages/kbn-eslint-plugin-license-header", "@kbn/plugin-generator": "link:packages/kbn-plugin-generator", "@kbn/test": "link:packages/kbn-test", + "@octokit/rest": "^15.10.0", "@types/angular": "^1.6.45", "@types/babel-core": "^6.25.5", "@types/bluebird": "^3.1.1", diff --git a/src/dev/failed_tests/report.js b/src/dev/failed_tests/report.js new file mode 100644 index 000000000000000..391c0696f095b2d --- /dev/null +++ b/src/dev/failed_tests/report.js @@ -0,0 +1,149 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import xml2js from 'xml2js'; +import vfs from 'vinyl-fs'; +import es from 'event-stream'; +import { getGithubClient, markdownMetadata, paginate } from '../github_utils'; +import { find } from 'lodash'; +import stripAnsi from 'strip-ansi'; + +const GITHUB_FLAKY_TEST_LABEL = 'flaky-failing-test'; +const GITHUB_OWNER = 'elastic'; +const GITHUB_REPO = 'kibana'; +const BUILD_URL = process.env.BUILD_URL; + +/** + * Parses junit XML files into JSON + */ +const mapXml = es.map((file, cb) => { + xml2js.parseString(file.contents.toString(), (err, result) => { + cb(null, result); + }); +}); + +/** + * Filters all testsuites to find failed testcases + */ +const filterFailures = es.map((testSuite, cb) => { + const testFiles = testSuite.testsuites.testsuite; + + const failures = testFiles.reduce((failures, testFile) => { + for (const testCase of testFile.testcase) { + if (testCase.failure) { + // unwrap xml weirdness + failures.push({ + ...testCase.$, + // Strip ANSI color characters + failure: stripAnsi(testCase.failure[0]) + }); + } + } + + return failures; + }, []); + + console.log(`Found ${failures.length} test failures`); + + cb(null, failures); +}); + +/** + * Creates and updates github issues for the given testcase failures. + */ +const updateGithubIssues = (githubClient, issues) => { + return es.map(async (failureCases, cb) => { + + const issueOps = failureCases.map(async (failureCase) => { + const existingIssue = find(issues, (issue) => { + return markdownMetadata.get(issue.body, 'test.class') === failureCase.classname && + markdownMetadata.get(issue.body, 'test.name') === failureCase.name; + }); + + if (existingIssue) { + // Increment failCount + const newCount = (markdownMetadata.get(existingIssue.body, 'test.failCount') || 0) + 1; + const newBody = markdownMetadata.set(existingIssue.body, 'test.failCount', newCount); + + await githubClient.issues.edit({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + number: existingIssue.number, + state: 'open', // Reopen issue if it was closed. + body: newBody + }); + + // Append a new comment + await githubClient.issues.createComment({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + number: existingIssue.number, + body: `New failure: [Jenkins Build](${BUILD_URL})` + }); + + console.log(`Updated issue ${existingIssue.html_url}, failCount: ${newCount}`); + } else { + let body = 'A test failed on a tracked branch\n' + + '```\n' + failureCase.failure + '\n```\n' + + `First failure: [Jenkins Build](${BUILD_URL})`; + body = markdownMetadata.set(body, { + 'test.class': failureCase.classname, + 'test.name': failureCase.name, + 'test.failCount': 1 + }); + + const newIssue = await githubClient.issues.create({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + title: `Failing test: ${failureCase.classname} - ${failureCase.name}`, + body: body, + labels: [GITHUB_FLAKY_TEST_LABEL] + }); + + console.log(`Created issue ${newIssue.data.html_url}`); + } + }); + + Promise + .all(issueOps) + .then(() => cb(null, failureCases)) + .catch(e => cb(e)); + }); +}; + +/** + * Scans all junit XML files in ./target/junit/ and reports any found test failures to Github Issues. + */ +export async function reportFailedTests(done) { + const githubClient = getGithubClient(); + const issues = await paginate(githubClient, githubClient.issues.getForRepo({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + labels: GITHUB_FLAKY_TEST_LABEL, + state: 'all', + per_page: 100 + })); + + vfs + .src(['./target/junit/**/*.xml']) + .pipe(mapXml) + .pipe(filterFailures) + .pipe(updateGithubIssues(githubClient, issues)) + .on('done', done); +} diff --git a/src/dev/github_utils/index.js b/src/dev/github_utils/index.js new file mode 100644 index 000000000000000..7c38f55e1f7e98d --- /dev/null +++ b/src/dev/github_utils/index.js @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Octokit from '@octokit/rest'; +import { markdownMetadata } from './metadata'; + +export { markdownMetadata }; + +export function getGithubClient() { + // const repo = await octokit.repos.getForOrg({ org: 'elastic', repo: 'kibana' }); + const client = new Octokit(); + client.authenticate({ + type: 'token', + token: process.env.GITHUB_TOKEN + }); + + return client; +} + +export async function paginate(client, promise) { + let response = await promise; + let { data } = response; + while (client.hasNextPage(response)) { + response = await client.getNextPage(response); + data = data.concat(response.data); + } + return data; +} diff --git a/src/dev/github_utils/metadata.js b/src/dev/github_utils/metadata.js new file mode 100644 index 000000000000000..f88f8c5bbc7340f --- /dev/null +++ b/src/dev/github_utils/metadata.js @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const REGEX = /\n\n/; + +/** + * Allows retrieving and setting key/value pairs on a Github Issue. Keys and values must be JSON-serializable. + * Borrowed heavily from https://github.com/probot/metadata/blob/6ae1523d5035ba727d09c0e7f77a6a154d9a4777/index.js + * + * `body` is a string that contains markdown and any existing metadata (eg. an issue or comment body) + * `prefix` is a string that can be used to namespace the metadata, defaults to `ci`. + */ +export const markdownMetadata = { + get(body, key = null, prefix = 'failed-test') { + const match = body.match(REGEX); + + if (match) { + const data = JSON.parse(match[1])[prefix]; + return key ? data && data[key] : data; + } else { + return null; + } + }, + + /** + * Set data on the body. Can either be set individually with `key` and `value` OR + */ + set(body, key, value, prefix = 'failed-test') { + let newData = {}; + // If second arg is an object, use all supplied values. + if (typeof key === 'object') { + newData = key; + prefix = value || prefix; // need to move third arg to prefix. + } else { + newData[key] = value; + } + + let data = {}; + + body = body.replace(REGEX, (_, json) => { + data = JSON.parse(json); + return ''; + }); + + if (!data[prefix]) data[prefix] = {}; + + Object.assign(data[prefix], newData); + + return `${body}\n\n`; + } +}; diff --git a/tasks/jenkins.js b/tasks/jenkins.js index e0f587e53bc9039..69f3bc8205934e6 100644 --- a/tasks/jenkins.js +++ b/tasks/jenkins.js @@ -17,6 +17,8 @@ * under the License. */ +import { reportFailedTests } from '../src/dev/failed_tests/report'; + module.exports = function (grunt) { grunt.registerTask('jenkins:docs', [ 'docker:docs' @@ -44,4 +46,12 @@ module.exports = function (grunt) { 'run:functionalTestsRelease', 'run:pluginFunctionalTestsRelease', ]); + + grunt.registerTask( + 'jenkins:report', + 'Reports failed tests found in junit xml files to Github issues', + function () { + reportFailedTests(this.async()); + } + ); }; diff --git a/test/scripts/jenkins_report_failed_tests.sh b/test/scripts/jenkins_report_failed_tests.sh new file mode 100755 index 000000000000000..c9e449d3304a120 --- /dev/null +++ b/test/scripts/jenkins_report_failed_tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e + +xvfb-run "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:report; diff --git a/yarn.lock b/yarn.lock index 25c1cab071a68d7..009252a9b28e88a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -126,6 +126,13 @@ version "0.2.3" resolved "https://registry.yarnpkg.com/@elastic/ui-ace/-/ui-ace-0.2.3.tgz#5281aed47a79b7216c55542b0675e435692f20cd" +"@gimenete/type-writer@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@gimenete/type-writer/-/type-writer-0.1.3.tgz#2d4f26118b18d71f5b34ca24fdd6d1fd455c05b6" + dependencies: + camelcase "^5.0.0" + prettier "^1.13.7" + "@kbn/babel-preset@link:packages/kbn-babel-preset": version "0.0.0" uid "" @@ -181,6 +188,20 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@octokit/rest@^15.10.0": + version "15.10.0" + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-15.10.0.tgz#9baf7430e55edf1a1024c35ae72ed2f5fc6e90e9" + dependencies: + "@gimenete/type-writer" "^0.1.3" + before-after-hook "^1.1.0" + btoa-lite "^1.0.0" + debug "^3.1.0" + http-proxy-agent "^2.1.0" + https-proxy-agent "^2.2.0" + lodash "^4.17.4" + node-fetch "^2.1.1" + url-template "^2.0.8" + "@samverschueren/stream-to-observable@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.0.tgz#ecdf48d532c58ea477acfcab80348424f8d0662f" @@ -1956,6 +1977,10 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +before-after-hook@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-1.1.0.tgz#83165e15a59460d13702cb8febd6a1807896db5a" + better-assert@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/better-assert/-/better-assert-1.0.2.tgz#40866b9e1b9e0b55b481894311e68faffaebc522" @@ -2267,6 +2292,10 @@ bser@^2.0.0: dependencies: node-int64 "^0.4.0" +btoa-lite@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337" + buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" @@ -2483,6 +2512,10 @@ camelcase@^4.0.0, camelcase@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd" +camelcase@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42" + caniuse-api@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-1.6.1.tgz#b534e7c734c4f81ec5fbe8aca2ad24354b962c6c" @@ -6377,7 +6410,7 @@ https-browserify@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" -https-proxy-agent@^2.2.1: +https-proxy-agent@^2.2.0, https-proxy-agent@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0" dependencies: @@ -9184,6 +9217,10 @@ node-fetch@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.1.tgz#369ca70b82f50c86496104a6c776d274f4e4a2d4" +node-fetch@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5" + node-gyp@^3.3.1: version "3.6.2" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.6.2.tgz#9bfbe54562286284838e750eac05295853fa1c60" @@ -10393,6 +10430,10 @@ preserve@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b" +prettier@^1.13.7: + version "1.14.2" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.14.2.tgz#0ac1c6e1a90baa22a62925f41963c841983282f9" + prettier@^1.14.0: version "1.14.0" resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.14.0.tgz#847c235522035fd988100f1f43cf20a7d24f9372" @@ -13630,6 +13671,10 @@ url-regex@^3.0.0: dependencies: ip-regex "^1.0.1" +url-template@^2.0.8: + version "2.0.8" + resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21" + url-to-options@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/url-to-options/-/url-to-options-1.0.1.tgz#1505a03a289a48cbd7a434efbaeec5055f5633a9"