From ee355747776ef6e62a807074f51e7d26165be12f Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Tue, 25 Jul 2023 15:17:18 +0200 Subject: [PATCH 1/7] test: run "websockets" example as a test --- .eslintrc | 3 +- examples/websockets/exampleRunner.js | 79 +++++++++++++++++++++++++++ examples/websockets/package-lock.json | 16 ++++-- examples/websockets/package.json | 5 +- examples/websockets/test.yml | 18 ++++++ examples/websockets/websockets.yml | 18 ------ 6 files changed, 112 insertions(+), 27 deletions(-) create mode 100755 examples/websockets/exampleRunner.js create mode 100644 examples/websockets/test.yml delete mode 100644 examples/websockets/websockets.yml diff --git a/.eslintrc b/.eslintrc index 83c6f192a6..ddf70069a2 100644 --- a/.eslintrc +++ b/.eslintrc @@ -5,7 +5,8 @@ "es6": true }, "globals": { - "expect": true + "expect": true, + "fetch": true }, "parserOptions": { "ecmaVersion": "latest", diff --git a/examples/websockets/exampleRunner.js b/examples/websockets/exampleRunner.js new file mode 100755 index 0000000000..3f08e54a20 --- /dev/null +++ b/examples/websockets/exampleRunner.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node +const { spawn } = require('node:child_process'); + +const [serverCommand, serverUrl, testCommand] = process.argv.slice(2); + +if (!serverCommand) { + throw new Error('Failed to run the test: server command missing'); +} + +if (!serverUrl) { + throw new Error('Failed to run the test: server URL is missing'); +} + +if (!testCommand) { + throw new Error('Failed to run the test: test command is missing'); +} + +async function run() { + // 1. Spawn the server process. + const serverProcess = spawn('npm', ['run', serverCommand], { + shell: '/bin/bash', + stdio: 'inherit' + }); + + serverProcess.on('error', (error) => { + console.error(error); + throw new Error( + 'Failed to spawn the test server. See the error output above.' + ); + }); + + const cleanup = () => { + if (!serverProcess.killed) { + serverProcess.kill(); + } + }; + + process + .on('SIGTERM', cleanup) + .on('SIGINT', cleanup) + .on('exit', cleanup) + .on('unhandledRejection', (error) => { + cleanup(); + throw error; + }); + + // 2. Ping the server to respond with anything but 5xx. + // This is the main difference from the "start-server-and-test" since + // the server of some of our scenarios won't respond with 200 + // (e.g. the WebSocket server responds with 426, rightfully). + await ping(serverUrl); + + // 3. Run the test command. + const testProcess = spawn('npm', ['run', testCommand], { + shell: '/bin/bash', + stdio: 'inherit' + }); + + testProcess.on('exit', cleanup); +} + +async function ping(url, retriesLeft = 3) { + return fetch(url, { method: 'HEAD' }) + .then((response) => response.status) + .catch(async () => { + retriesLeft--; + + if (retriesLeft === 0) { + throw new Error( + `Failed to ping the server at "${url}": server did not respond` + ); + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + return ping(url, retriesLeft); + }); +} + +run(); diff --git a/examples/websockets/package-lock.json b/examples/websockets/package-lock.json index c24533084d..31846dbb50 100644 --- a/examples/websockets/package-lock.json +++ b/examples/websockets/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "websockets", "version": "1.0.0", "license": "ISC", "dependencies": { @@ -1516,9 +1517,12 @@ } }, "node_modules/minimist": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/moment": { "version": "2.29.4", @@ -3593,9 +3597,9 @@ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" }, "minimist": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" }, "moment": { "version": "2.29.4", diff --git a/examples/websockets/package.json b/examples/websockets/package.json index dfeab6e079..60d72dec67 100644 --- a/examples/websockets/package.json +++ b/examples/websockets/package.json @@ -4,9 +4,10 @@ "description": "Artillery.io - WebSockets example", "main": "app.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "start": "./testRunner.js server http://localhost:8888 test", + "server": "node app.js", + "test": "artillery run test.yml" }, - "author": "", "license": "ISC", "dependencies": { "artillery": "^1.7.2", diff --git a/examples/websockets/test.yml b/examples/websockets/test.yml new file mode 100644 index 0000000000..d05544729a --- /dev/null +++ b/examples/websockets/test.yml @@ -0,0 +1,18 @@ +config: + target: 'ws://localhost:8888/' + processor: './my-functions.js' + phases: + - duration: 60 + arrivalRate: 25 + +scenarios: + - name: 'Sending a string' + engine: ws + flow: + - send: 'Artillery' + + - name: 'Sending an object from a function' + engine: ws + flow: + - function: 'createRandomScore' + - send: '{{ data }}' diff --git a/examples/websockets/websockets.yml b/examples/websockets/websockets.yml deleted file mode 100644 index a14947da34..0000000000 --- a/examples/websockets/websockets.yml +++ /dev/null @@ -1,18 +0,0 @@ -config: - target: "ws://localhost:8888/" - processor: "./my-functions.js" - phases: - - duration: 60 - arrivalRate: 25 - -scenarios: - - name: "Sending a string" - engine: ws - flow: - - send: "Artillery" - - - name: "Sending an object from a function" - engine: ws - flow: - - function: "createRandomScore" - - send: "{{ data }}" From 02f61deb58ff8f26d6b4e3c085b8e17cde08ab6e Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Tue, 25 Jul 2023 15:25:36 +0200 Subject: [PATCH 2/7] chore: add a reusable example test workflow chore: set job timeout to 60 minutes chore: define workflow next to the example --- .github/workflows/examples.yml | 20 +++++++++++++++ examples/websockets/.github/workflows/ci.yml | 26 ++++++++++++++++++++ examples/websockets/package.json | 2 +- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/examples.yml create mode 100644 examples/websockets/.github/workflows/ci.yml diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml new file mode 100644 index 0000000000..a36c3e34e3 --- /dev/null +++ b/.github/workflows/examples.yml @@ -0,0 +1,20 @@ +name: Examples + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + prepare: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Prepare examples + run: cp ./examples/websockets/.github/workflows/ci.yml ./.github/workflows/example-websockets.yml + + websockets: + uses: ./.github/workflows/example-websockets.yml diff --git a/examples/websockets/.github/workflows/ci.yml b/examples/websockets/.github/workflows/ci.yml new file mode 100644 index 0000000000..086a58fbb6 --- /dev/null +++ b/examples/websockets/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: test-websockets + +on: + push: + branches: [main] + workflow_call: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 60 + defaults: + run: + working-directory: ./examples/websockets + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install dependencies + run: npm ci + + - name: Run load tests + uses: artilleryio/action-cli@v1 + with: + command: run test.yml diff --git a/examples/websockets/package.json b/examples/websockets/package.json index 60d72dec67..8ff111c933 100644 --- a/examples/websockets/package.json +++ b/examples/websockets/package.json @@ -4,7 +4,7 @@ "description": "Artillery.io - WebSockets example", "main": "app.js", "scripts": { - "start": "./testRunner.js server http://localhost:8888 test", + "start": "./exampleRunner.js server http://localhost:8888 test", "server": "node app.js", "test": "artillery run test.yml" }, From 88f67735a33b1eacef37373b22b1765a7f759231 Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Fri, 4 Aug 2023 11:28:52 +0200 Subject: [PATCH 3/7] chore: run metrics example --- .github/workflows/examples.yml | 20 ++++++++++----- examples/websockets/.github/workflows/ci.yml | 26 -------------------- 2 files changed, 14 insertions(+), 32 deletions(-) delete mode 100644 examples/websockets/.github/workflows/ci.yml diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index a36c3e34e3..e936d9e86e 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -7,14 +7,22 @@ on: workflow_dispatch: jobs: - prepare: + http-metrics-by-endpoint: runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CWD: ./examples/http-metrics-by-endpoint + defaults: + run: + working-directory: ${{ env.CWD }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Prepare examples - run: cp ./examples/websockets/.github/workflows/ci.yml ./.github/workflows/example-websockets.yml + - name: Install + run: npm ci - websockets: - uses: ./.github/workflows/example-websockets.yml + - name: Test + uses: artilleryio/action-cli@v1 + with: + command: run ${{ env.CWD }}/endpoint-metrics.yml diff --git a/examples/websockets/.github/workflows/ci.yml b/examples/websockets/.github/workflows/ci.yml deleted file mode 100644 index 086a58fbb6..0000000000 --- a/examples/websockets/.github/workflows/ci.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: test-websockets - -on: - push: - branches: [main] - workflow_call: - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 60 - defaults: - run: - working-directory: ./examples/websockets - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install dependencies - run: npm ci - - - name: Run load tests - uses: artilleryio/action-cli@v1 - with: - command: run test.yml From e4d5752b214bb1d1713d54b13468bdec7a4e4cd1 Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Fri, 4 Aug 2023 12:21:34 +0200 Subject: [PATCH 4/7] test: add "multiple-scenario-specs" test --- .github/workflows/examples.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index e936d9e86e..2cfca1d708 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -26,3 +26,28 @@ jobs: uses: artilleryio/action-cli@v1 with: command: run ${{ env.CWD }}/endpoint-metrics.yml + + multiple-scenarios-spec: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CWD: ./examples/multiple-scenario-specs + defaults: + run: + working-directory: ${{ env.CWD }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install + run: npm ci + + - name: Run armadillo scenario + uses: artilleryio/action-cli@v1 + with: + command: run --config ${{ env.CWD }}/common-config.yml ${{ env.CWD }}/scenarios/armadillo.yml + + - name: Run dino scenario + uses: artilleryio/action-cli@v1 + with: + command: run --config ${{ env.CWD }}/common-config.yml ${{ env.CWD }}/scenarios/dino.yml From ebb27b187e3ecf25f59ab574749eb65f4ec3ba18 Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Fri, 4 Aug 2023 12:25:58 +0200 Subject: [PATCH 5/7] test: add "using-data-from-csv" test --- .github/workflows/examples.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 2cfca1d708..33a0b1dbbf 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -51,3 +51,23 @@ jobs: uses: artilleryio/action-cli@v1 with: command: run --config ${{ env.CWD }}/common-config.yml ${{ env.CWD }}/scenarios/dino.yml + + using-data-from-csv: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CWD: ./examples/using-data-from-csv + defaults: + run: + working-directory: ${{ env.CWD }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install + run: npm ci + + - name: Test + uses: artilleryio/action-cli@v1 + with: + command: run ${{ env.CWD }}/website-test.yml From c8b0d120b9f77271032a851860edd24d77293a4f Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Fri, 4 Aug 2023 12:46:45 +0200 Subject: [PATCH 6/7] chore: cleanup websockets test setup chore: patch artillery binary chore: remove install steps from tests chore: set custom "working-directory" for tests --- .github/workflows/examples.yml | 21 +++----- examples/websockets/README.md | 2 +- examples/websockets/exampleRunner.js | 79 ---------------------------- examples/websockets/package.json | 1 - 4 files changed, 9 insertions(+), 94 deletions(-) delete mode 100755 examples/websockets/exampleRunner.js diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 33a0b1dbbf..68e9593bd4 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -19,13 +19,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Install - run: npm ci - - name: Test uses: artilleryio/action-cli@v1 with: - command: run ${{ env.CWD }}/endpoint-metrics.yml + command: run ./endpoint-metrics.yml + working-directory: ${{ env.CWD }} multiple-scenarios-spec: runs-on: ubuntu-latest @@ -39,18 +37,17 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Install - run: npm ci - - name: Run armadillo scenario uses: artilleryio/action-cli@v1 with: - command: run --config ${{ env.CWD }}/common-config.yml ${{ env.CWD }}/scenarios/armadillo.yml + command: run --config ./common-config.yml ./scenarios/armadillo.yml + working-directory: ${{ env.CWD }} - name: Run dino scenario uses: artilleryio/action-cli@v1 with: - command: run --config ${{ env.CWD }}/common-config.yml ${{ env.CWD }}/scenarios/dino.yml + command: run --config ./common-config.yml ./scenarios/dino.yml + working-directory: ${{ env.CWD }} using-data-from-csv: runs-on: ubuntu-latest @@ -64,10 +61,8 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Install - run: npm ci - - name: Test uses: artilleryio/action-cli@v1 with: - command: run ${{ env.CWD }}/website-test.yml + command: run ./website-test.yml + working-directory: ${{ env.CWD }} diff --git a/examples/websockets/README.md b/examples/websockets/README.md index 3813b021b4..4c19ba9263 100644 --- a/examples/websockets/README.md +++ b/examples/websockets/README.md @@ -13,7 +13,7 @@ npm install After installing the dependencies, start the WebSockets server: ``` -node app.js +npm run server ``` This command will start a WebSockets server listening at ws://localhost:8888. diff --git a/examples/websockets/exampleRunner.js b/examples/websockets/exampleRunner.js deleted file mode 100755 index 3f08e54a20..0000000000 --- a/examples/websockets/exampleRunner.js +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env node -const { spawn } = require('node:child_process'); - -const [serverCommand, serverUrl, testCommand] = process.argv.slice(2); - -if (!serverCommand) { - throw new Error('Failed to run the test: server command missing'); -} - -if (!serverUrl) { - throw new Error('Failed to run the test: server URL is missing'); -} - -if (!testCommand) { - throw new Error('Failed to run the test: test command is missing'); -} - -async function run() { - // 1. Spawn the server process. - const serverProcess = spawn('npm', ['run', serverCommand], { - shell: '/bin/bash', - stdio: 'inherit' - }); - - serverProcess.on('error', (error) => { - console.error(error); - throw new Error( - 'Failed to spawn the test server. See the error output above.' - ); - }); - - const cleanup = () => { - if (!serverProcess.killed) { - serverProcess.kill(); - } - }; - - process - .on('SIGTERM', cleanup) - .on('SIGINT', cleanup) - .on('exit', cleanup) - .on('unhandledRejection', (error) => { - cleanup(); - throw error; - }); - - // 2. Ping the server to respond with anything but 5xx. - // This is the main difference from the "start-server-and-test" since - // the server of some of our scenarios won't respond with 200 - // (e.g. the WebSocket server responds with 426, rightfully). - await ping(serverUrl); - - // 3. Run the test command. - const testProcess = spawn('npm', ['run', testCommand], { - shell: '/bin/bash', - stdio: 'inherit' - }); - - testProcess.on('exit', cleanup); -} - -async function ping(url, retriesLeft = 3) { - return fetch(url, { method: 'HEAD' }) - .then((response) => response.status) - .catch(async () => { - retriesLeft--; - - if (retriesLeft === 0) { - throw new Error( - `Failed to ping the server at "${url}": server did not respond` - ); - } - - await new Promise((resolve) => setTimeout(resolve, 500)); - return ping(url, retriesLeft); - }); -} - -run(); diff --git a/examples/websockets/package.json b/examples/websockets/package.json index 8ff111c933..0083abd181 100644 --- a/examples/websockets/package.json +++ b/examples/websockets/package.json @@ -4,7 +4,6 @@ "description": "Artillery.io - WebSockets example", "main": "app.js", "scripts": { - "start": "./exampleRunner.js server http://localhost:8888 test", "server": "node app.js", "test": "artillery run test.yml" }, From 19e4f37de58bb74b3545f03d8ee208e872201a37 Mon Sep 17 00:00:00 2001 From: Artem Zakharchenko Date: Mon, 7 Aug 2023 12:29:21 +0200 Subject: [PATCH 7/7] chore: use local artillery binary chore: use absolute artillery binary path chore: add install step to use custom binary --- .github/workflows/examples.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 68e9593bd4..91bc3d00fc 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -6,6 +6,9 @@ on: pull_request: workflow_dispatch: +env: + ARTILLERY_BINARY_PATH: ${{ github.workspace }}/packages/artillery/bin/run + jobs: http-metrics-by-endpoint: runs-on: ubuntu-latest @@ -19,11 +22,16 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Install + run: npm ci + - name: Test uses: artilleryio/action-cli@v1 with: command: run ./endpoint-metrics.yml working-directory: ${{ env.CWD }} + env: + ARTILLERY_BINARY_PATH: ${{ env.ARTILLERY_BINARY_PATH }} multiple-scenarios-spec: runs-on: ubuntu-latest @@ -37,17 +45,24 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Install + run: npm ci + - name: Run armadillo scenario uses: artilleryio/action-cli@v1 with: command: run --config ./common-config.yml ./scenarios/armadillo.yml working-directory: ${{ env.CWD }} + env: + ARTILLERY_BINARY_PATH: ${{ env.ARTILLERY_BINARY_PATH }} - name: Run dino scenario uses: artilleryio/action-cli@v1 with: command: run --config ./common-config.yml ./scenarios/dino.yml working-directory: ${{ env.CWD }} + env: + ARTILLERY_BINARY_PATH: ${{ env.ARTILLERY_BINARY_PATH }} using-data-from-csv: runs-on: ubuntu-latest @@ -61,8 +76,13 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Install + run: npm ci + - name: Test uses: artilleryio/action-cli@v1 with: command: run ./website-test.yml working-directory: ${{ env.CWD }} + env: + ARTILLERY_BINARY_PATH: ${{ env.ARTILLERY_BINARY_PATH }}