Skip to content

Commit

Permalink
feat(CICD): handling E2E test results in order to measure coverage (#…
Browse files Browse the repository at this point in the history
…30149)

Also make sure results are shown in the `Test Report` job.

Refs: #29848
  • Loading branch information
victoralfaro-dotcms authored Sep 30, 2024
1 parent ec4a905 commit 43cd90a
Show file tree
Hide file tree
Showing 9 changed files with 397 additions and 20 deletions.
3 changes: 1 addition & 2 deletions .github/workflows/cicd_2-merge-queue.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ jobs:
postman: ${{ needs.initialize.outputs.backend == 'true' }}
frontend: ${{ needs.initialize.outputs.frontend == 'true' }}
cli: ${{ needs.initialize.outputs.cli == 'true' }}
# This will probably need to be re-evaluated, that is if it makes sense to tun E2E tests at the merge queue
# e2e: ${{ needs.initialize.outputs.build }}
e2e: ${{ needs.initialize.outputs.build == 'true' }}
secrets:
DOTCMS_LICENSE: ${{ secrets.DOTCMS_LICENSE }}
finalize:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/cicd_comp_test-phase.yml
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,8 @@ jobs:
fail-fast: false
matrix:
suites:
- { name: "JVM E2E Suite", pathName: "jvme2etestsuite", maven_args: '-Dit.test=E2eTestSuite -Dci=true -pl :dotcms-e2e-java' }
- { name: "Node.js E2E Suite", pathName: "nodee2etestsuite", maven_args: '-De2e.test.env=ci -pl :dotcms-e2e-node' }
- { name: "JVM E2E Suite", pathName: "jvme2etestsuite", maven_args: '-Dci=true -Dit.test=E2eTestSuite -De2e.test.forkCount=1 -pl :dotcms-e2e-java' }
- { name: "Node E2E Suite", pathName: "nodee2etestsuite", maven_args: '-De2e.test.env=ci -pl :dotcms-e2e-node' }
steps:
- name: Checkout code
uses: actions/checkout@v4
Expand Down
3 changes: 0 additions & 3 deletions dotcms-postman/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
<docker.wm.volume>./src/test/resources</docker.wm.volume>
<docker.wm.volume.internal>/home/wiremock</docker.wm.volume.internal>
<testdata.dir>${project.build.directory}/testdata</testdata.dir>
<it.test.fork-folder.prefix>${project.build.directory}/testdata/fork-</it.test.fork-folder.prefix>
<it.test.fork-folder>${it.test.fork-folder.prefix}</it.test.fork-folder>
<cleanup.before.tests>true</cleanup.before.tests>
<postman.test.skip>true</postman.test.skip>
<yarn.install.cmd>install --frozen-lockfile</yarn.install.cmd>
Expand Down Expand Up @@ -136,7 +134,6 @@
<DOT_AI_IMAGE_API_URL>http://wm:8080/i</DOT_AI_IMAGE_API_URL>
<DOT_AI_EMBEDDINGS_API_URL>http://wm:8080/e</DOT_AI_EMBEDDINGS_API_URL>
<DOT_AI_MODELS_API_URL>http://wm:8080/m</DOT_AI_MODELS_API_URL>
<DOT_AI_DEBUG_LOGGER>true</DOT_AI_DEBUG_LOGGER>
</env>
<links combine.children="append">
<link>wiremock:wm</link>
Expand Down
4 changes: 3 additions & 1 deletion e2e/dotcms-e2e-java/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@
<docker.image.wiremock>${ext.docker.image.wiremock}</docker.image.wiremock>
<docker.wm.volume>./src/test/resources</docker.wm.volume>
<docker.wm.volume.internal>/home/wiremock</docker.wm.volume.internal>
<docker.jacoco.skip>false</docker.jacoco.skip>
<wiremock.port>50505</wiremock.port>
<wiremock.api.key>${ext.wiremock.api.key}</wiremock.api.key>
<tomcat.port>8080</tomcat.port>
<docker.jacoco.skip>false</docker.jacoco.skip>
</properties>

<dependencies>
Expand Down Expand Up @@ -170,6 +170,8 @@
<argument>-e</argument>
<argument>-Dexec.mainClass=com.microsoft.playwright.CLI</argument>
<argument>-Dexec.args="install-deps"</argument>
<argument>-pl</argument>
<argument>:dotcms-e2e-java</argument>
</arguments>
</configuration>
</execution>
Expand Down
84 changes: 84 additions & 0 deletions e2e/dotcms-e2e-node/frontend/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
const fs = require('fs');
const path = require('path');
const xml2js = require('xml2js');

const summaryResults = {
completed: 0,
errors: 0,
failures: 0,
skipped: 0
};

// Default configurations
const defaultE2eTestsResultsDir = '../target/failsafe-reports'; // Default results directory

// Function to generate failsafe-summary.xml
const generateFailsafeSummaryXml = (e2eTestsResultsDir) => {
const builder = new xml2js.Builder();
const result = summaryResults.failures > 0 ? 255 : 0;
const xmlObj = {
'failsafe-summary': {
$: { result: result },
completed: summaryResults.completed,
errors: summaryResults.errors,
failures: summaryResults.failures,
skipped: summaryResults.skipped
}
};

if (summaryResults.failures > 0) {
xmlObj['failsafe-summary']['failureMessage'] = 'There are test failures.';
}

const xml = builder.buildObject(xmlObj);

fs.writeFileSync(path.join(e2eTestsResultsDir, 'failsafe-summary.xml'), xml, 'utf8');
}

const processTestsResults = async () => {
const parser = new xml2js.Parser();
const xml = fs.readFileSync(process.env.PLAYWRIGHT_JUNIT_OUTPUT_FILE, 'utf8');
const xmlDoc = await parser.parseStringPromise(xml);

xmlDoc.testsuites.testsuite.forEach((ts) => {
const tests = parseInt(ts.tests || '0');
const errors = parseInt(ts.errors || '0');
const failures = parseInt(ts.failures || '0');
const skipped = parseInt(ts.skipped || '0');
summaryResults.completed += tests;
summaryResults.errors += errors;
summaryResults.failures += failures;
summaryResults.skipped += skipped;
});
};

/**
* Main function to process command line arguments and run the script.
* Parses command line arguments for named parameters.
*/
async function main() {
const e2eTestsResultsDir = path.resolve(__dirname, defaultE2eTestsResultsDir);
// Ensure results directory exists
if (!fs.existsSync(e2eTestsResultsDir)) {
fs.mkdirSync(e2eTestsResultsDir, { recursive: true });
}

try {
await processTestsResults();
generateFailsafeSummaryXml(e2eTestsResultsDir);

if (summaryResults.failures > 0) {
console.error('Some E2E tests failed.');
process.exit(0);
}
} catch (error) {
console.error('An error occurred:', error);
process.exit(1);
}
}

// Run the main function and handle any errors
main().catch((err) => {
console.error('Unhandled error:', err);
process.exit(1);
});
10 changes: 8 additions & 2 deletions e2e/dotcms-e2e-node/frontend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,14 @@
"@types/node": "^22.5.4",
"dotenv": "^16.4.5"
},
"dependencies": {
"jsdom": "^25.0.1",
"xml2js": "^0.6.2"
},
"scripts": {
"start-local": "CURRENT_ENV=local yarn playwright test",
"start-ci": "CURRENT_ENV=ci yarn playwright test"
"start": "PLAYWRIGHT_JUNIT_SUITE_ID=nodee2etestsuite PLAYWRIGHT_JUNIT_SUITE_NAME='E2E Node Test Suite' PLAYWRIGHT_JUNIT_OUTPUT_FILE='../target/failsafe-reports/TEST-e2e-node-results.xml' yarn playwright test",
"start-local": "CURRENT_ENV=local yarn run start",
"start-ci": "CURRENT_ENV=ci yarn run start",
"post-testing": "PLAYWRIGHT_JUNIT_OUTPUT_FILE='../target/failsafe-reports/TEST-e2e-node-results.xml' node index.js"
}
}
11 changes: 4 additions & 7 deletions e2e/dotcms-e2e-node/frontend/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import dotenv from 'dotenv';
import * as path from "node:path";
import { defineConfig, devices } from '@playwright/test';


const resolveEnvs = () => {
const envFiles = ['.env'];

Expand All @@ -20,12 +19,7 @@ const resolveEnvs = () => {
});
};

const prepareForTesting = () => {

};

resolveEnvs();
prepareForTesting();

/**
* See https://playwright.dev/docs/test-configuration.
Expand All @@ -41,7 +35,10 @@ export default defineConfig({
/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: [['junit', { outputFile: '../target/failsafe-reports/failsafe-summary.xml' }]],
reporter: [
['junit'],
['github']
],
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
baseURL: process.env.BASE_URL,
Expand Down
Loading

0 comments on commit 43cd90a

Please sign in to comment.