-
Notifications
You must be signed in to change notification settings - Fork 8.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
321 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,149 @@ | ||
/* | ||
* Licensed to Elasticsearch B.V. under one or more contributor | ||
* license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright | ||
* ownership. Elasticsearch B.V. licenses this file to you under | ||
* the Apache License, Version 2.0 (the "License"); you may | ||
* not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
import xml2js from 'xml2js'; | ||
import vfs from 'vinyl-fs'; | ||
import es from 'event-stream'; | ||
import { getGithubClient, markdownMetadata, paginate } from '../github_utils'; | ||
import { find } from 'lodash'; | ||
import stripAnsi from 'strip-ansi'; | ||
|
||
const GITHUB_FLAKY_TEST_LABEL = 'flaky-failing-test'; | ||
const GITHUB_OWNER = 'elastic'; | ||
const GITHUB_REPO = 'kibana'; | ||
const BUILD_URL = process.env.BUILD_URL; | ||
|
||
/** | ||
* Parses junit XML files into JSON | ||
*/ | ||
const mapXml = es.map((file, cb) => { | ||
xml2js.parseString(file.contents.toString(), (err, result) => { | ||
cb(null, result); | ||
}); | ||
}); | ||
|
||
/** | ||
* Filters all testsuites to find failed testcases | ||
*/ | ||
const filterFailures = es.map((testSuite, cb) => { | ||
const testFiles = testSuite.testsuites.testsuite; | ||
|
||
const failures = testFiles.reduce((failures, testFile) => { | ||
for (const testCase of testFile.testcase) { | ||
if (testCase.failure) { | ||
// unwrap xml weirdness | ||
failures.push({ | ||
...testCase.$, | ||
// Strip ANSI color characters | ||
failure: stripAnsi(testCase.failure[0]) | ||
}); | ||
} | ||
} | ||
|
||
return failures; | ||
}, []); | ||
|
||
console.log(`Found ${failures.length} test failures`); | ||
|
||
cb(null, failures); | ||
}); | ||
|
||
/** | ||
* Creates and updates github issues for the given testcase failures. | ||
*/ | ||
const updateGithubIssues = (githubClient, issues) => { | ||
return es.map(async (failureCases, cb) => { | ||
|
||
const issueOps = failureCases.map(async (failureCase) => { | ||
const existingIssue = find(issues, (issue) => { | ||
return markdownMetadata.get(issue.body, 'test.class') === failureCase.classname && | ||
markdownMetadata.get(issue.body, 'test.name') === failureCase.name; | ||
}); | ||
|
||
if (existingIssue) { | ||
// Increment failCount | ||
const newCount = (markdownMetadata.get(existingIssue.body, 'test.failCount') || 0) + 1; | ||
const newBody = markdownMetadata.set(existingIssue.body, 'test.failCount', newCount); | ||
|
||
await githubClient.issues.edit({ | ||
owner: GITHUB_OWNER, | ||
repo: GITHUB_REPO, | ||
number: existingIssue.number, | ||
state: 'open', // Reopen issue if it was closed. | ||
body: newBody | ||
}); | ||
|
||
// Append a new comment | ||
await githubClient.issues.createComment({ | ||
owner: GITHUB_OWNER, | ||
repo: GITHUB_REPO, | ||
number: existingIssue.number, | ||
body: `New failure: [Jenkins Build](${BUILD_URL})` | ||
}); | ||
|
||
console.log(`Updated issue ${existingIssue.html_url}, failCount: ${newCount}`); | ||
} else { | ||
let body = 'A test failed on a tracked branch\n' + | ||
'```\n' + failureCase.failure + '\n```\n' + | ||
`First failure: [Jenkins Build](${BUILD_URL})`; | ||
body = markdownMetadata.set(body, { | ||
'test.class': failureCase.classname, | ||
'test.name': failureCase.name, | ||
'test.failCount': 1 | ||
}); | ||
|
||
const newIssue = await githubClient.issues.create({ | ||
owner: GITHUB_OWNER, | ||
repo: GITHUB_REPO, | ||
title: `Failing test: ${failureCase.classname} - ${failureCase.name}`, | ||
body: body, | ||
labels: [GITHUB_FLAKY_TEST_LABEL] | ||
}); | ||
|
||
console.log(`Created issue ${newIssue.data.html_url}`); | ||
} | ||
}); | ||
|
||
Promise | ||
.all(issueOps) | ||
.then(() => cb(null, failureCases)) | ||
.catch(e => cb(e)); | ||
}); | ||
}; | ||
|
||
/** | ||
* Scans all junit XML files in ./target/junit/ and reports any found test failures to Github Issues. | ||
*/ | ||
export async function reportFailedTests(done) { | ||
const githubClient = getGithubClient(); | ||
const issues = await paginate(githubClient, githubClient.issues.getForRepo({ | ||
owner: GITHUB_OWNER, | ||
repo: GITHUB_REPO, | ||
labels: GITHUB_FLAKY_TEST_LABEL, | ||
state: 'all', | ||
per_page: 100 | ||
})); | ||
|
||
vfs | ||
.src(['./target/junit/**/*.xml']) | ||
.pipe(mapXml) | ||
.pipe(filterFailures) | ||
.pipe(updateGithubIssues(githubClient, issues)) | ||
.on('done', done); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
/* | ||
* Licensed to Elasticsearch B.V. under one or more contributor | ||
* license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright | ||
* ownership. Elasticsearch B.V. licenses this file to you under | ||
* the Apache License, Version 2.0 (the "License"); you may | ||
* not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
import Octokit from '@octokit/rest'; | ||
import { markdownMetadata } from './metadata'; | ||
|
||
export { markdownMetadata }; | ||
|
||
export function getGithubClient() { | ||
const client = new Octokit(); | ||
client.authenticate({ | ||
type: 'token', | ||
token: process.env.GITHUB_TOKEN | ||
}); | ||
|
||
return client; | ||
} | ||
|
||
export async function paginate(client, promise) { | ||
let response = await promise; | ||
let { data } = response; | ||
while (client.hasNextPage(response)) { | ||
response = await client.getNextPage(response); | ||
data = data.concat(response.data); | ||
} | ||
return data; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
/* | ||
* Licensed to Elasticsearch B.V. under one or more contributor | ||
* license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright | ||
* ownership. Elasticsearch B.V. licenses this file to you under | ||
* the Apache License, Version 2.0 (the "License"); you may | ||
* not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
const REGEX = /\n\n<!-- kibanaCiData = (.*) -->/; | ||
|
||
/** | ||
* Allows retrieving and setting key/value pairs on a Github Issue. Keys and values must be JSON-serializable. | ||
* Borrowed heavily from https://github.com/probot/metadata/blob/6ae1523d5035ba727d09c0e7f77a6a154d9a4777/index.js | ||
* | ||
* `body` is a string that contains markdown and any existing metadata (eg. an issue or comment body) | ||
* `prefix` is a string that can be used to namespace the metadata, defaults to `ci`. | ||
*/ | ||
export const markdownMetadata = { | ||
get(body, key = null, prefix = 'failed-test') { | ||
const match = body.match(REGEX); | ||
|
||
if (match) { | ||
const data = JSON.parse(match[1])[prefix]; | ||
return key ? data && data[key] : data; | ||
} else { | ||
return null; | ||
} | ||
}, | ||
|
||
/** | ||
* Set data on the body. Can either be set individually with `key` and `value` OR | ||
*/ | ||
set(body, key, value, prefix = 'failed-test') { | ||
let newData = {}; | ||
// If second arg is an object, use all supplied values. | ||
if (typeof key === 'object') { | ||
newData = key; | ||
prefix = value || prefix; // need to move third arg to prefix. | ||
} else { | ||
newData[key] = value; | ||
} | ||
|
||
let data = {}; | ||
|
||
body = body.replace(REGEX, (_, json) => { | ||
data = JSON.parse(json); | ||
return ''; | ||
}); | ||
|
||
if (!data[prefix]) data[prefix] = {}; | ||
|
||
Object.assign(data[prefix], newData); | ||
|
||
return `${body}\n\n<!-- kibanaCiData = ${JSON.stringify(data)} -->`; | ||
} | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -e | ||
|
||
xvfb-run "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:report; |
Oops, something went wrong.