From 851147d0dea917e6852751cb2914ca1bb8287394 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 24 Jul 2024 17:13:09 -0400 Subject: [PATCH 01/12] Expose Cypress tag environment variable to Docker containers --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 229dd704059..3d0c07fd04e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,7 @@ x-e2e-env: # Cloud Manager-specific test configuration. CY_TEST_SUITE: ${CY_TEST_SUITE} CY_TEST_REGION: ${CY_TEST_REGION} + CY_TEST_TAGS: ${CY_TEST_TAGS} # Cypress environment variables for alternative parallelization. CY_TEST_SPLIT_RUN: ${CY_TEST_SPLIT_RUN} From 211118dbf2d3d5b3d40164e6004eeea0ced3a0ee Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 24 Jul 2024 17:15:24 -0400 Subject: [PATCH 02/12] Add e2e_heimdall Docker Compose service for Heimdall pipeline --- docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 3d0c07fd04e..d27b0025134 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,12 +105,23 @@ services: timeout: 10s retries: 10 + # Generic end-to-end test runner for Cloud's primary testing pipeline. + # Configured to run against a local Cloud instance. e2e: <<: *default-runner environment: <<: *default-env MANAGER_OAUTH: ${MANAGER_OAUTH} + # End-to-end test runner for Cloud's synthetic monitoring tests. + # Configured to run against a remote Cloud instance hosted at some URL. + e2e_heimdall: + <<: *default-runner + depends_on: ~ + environment: + <<: *default-env + MANAGER_OAUTH: ${MANAGER_OAUTH} + region-1: build: context: . From bced338b3f46176b617737cb1e6732c40fe6ed88 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 24 Jul 2024 17:18:39 -0400 Subject: [PATCH 03/12] Make 'depends_on' an empty list for e2e_heidmall service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d27b0025134..3279fc516e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,7 +117,7 @@ services: # Configured to run against a remote Cloud instance hosted at some URL. e2e_heimdall: <<: *default-runner - depends_on: ~ + depends_on: environment: <<: *default-env MANAGER_OAUTH: ${MANAGER_OAUTH} From 2e21e999ee7148a282113a03703bd170db9fe3bb Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 24 Jul 2024 17:23:04 -0400 Subject: [PATCH 04/12] Make empty list more explicit to resolve Docker error --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3279fc516e2..41d4c594fe4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,7 +117,7 @@ services: # Configured to run against a remote Cloud instance hosted at some URL. e2e_heimdall: <<: *default-runner - depends_on: + depends_on: [] environment: <<: *default-env MANAGER_OAUTH: ${MANAGER_OAUTH} From a833497c9979c78168e0520853eb368e7cd5a766 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Wed, 24 Jul 2024 17:43:28 -0400 Subject: [PATCH 05/12] Tag tests for synthetic testing --- .../cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts | 2 ++ packages/manager/cypress/e2e/core/volumes/create-volume.spec.ts | 2 ++ packages/manager/cypress/support/util/tag.ts | 1 + 3 files changed, 5 insertions(+) diff --git a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts index 4d415a8fb19..3c6b63c6ec0 100644 --- a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts +++ b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts @@ -156,6 +156,8 @@ describe('object storage end-to-end tests', () => { * - Confirms that deleted buckets are no longer listed on landing page. */ it('can create and delete object storage buckets', () => { + cy.tag('purpose:syntheticTesting'); + const bucketLabel = randomLabel(); const bucketRegion = 'Atlanta, GA'; const bucketCluster = 'us-southeast-1'; diff --git a/packages/manager/cypress/e2e/core/volumes/create-volume.spec.ts b/packages/manager/cypress/e2e/core/volumes/create-volume.spec.ts index 3041575a061..25115abf26e 100644 --- a/packages/manager/cypress/e2e/core/volumes/create-volume.spec.ts +++ b/packages/manager/cypress/e2e/core/volumes/create-volume.spec.ts @@ -27,6 +27,8 @@ describe('volume create flow', () => { * - Confirms that volume is listed correctly on volumes landing page. */ it('creates an unattached volume', () => { + cy.tag('purpose:syntheticTesting'); + const region = chooseRegion(); const volume = { label: randomLabel(), diff --git a/packages/manager/cypress/support/util/tag.ts b/packages/manager/cypress/support/util/tag.ts index c7fa37f0a08..9be33d52a39 100644 --- a/packages/manager/cypress/support/util/tag.ts +++ b/packages/manager/cypress/support/util/tag.ts @@ -18,6 +18,7 @@ export type TestTag = // DC testing purposes even if that is not the primary purpose of the test. | 'purpose:dcTesting' | 'purpose:smokeTesting' + | 'purpose:syntheticTesting' // Method-related tags. // Describe the way the tests operate -- either end-to-end using real API requests, From c0f98a95dc42f613b2b2b473132818cd5710e16e Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 25 Jul 2024 10:16:13 -0400 Subject: [PATCH 06/12] Temporarily force test to fail --- .../cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts index 3c6b63c6ec0..b2841726723 100644 --- a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts +++ b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts @@ -157,6 +157,7 @@ describe('object storage end-to-end tests', () => { */ it('can create and delete object storage buckets', () => { cy.tag('purpose:syntheticTesting'); + throw new Error('Fail'); const bucketLabel = randomLabel(); const bucketRegion = 'Atlanta, GA'; From 660e3f2afdaa8a1aad02d724901737a6f0ccd53c Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 25 Jul 2024 11:33:47 -0400 Subject: [PATCH 07/12] Stop test from failing --- .../cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts index b2841726723..3c6b63c6ec0 100644 --- a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts +++ b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts @@ -157,7 +157,6 @@ describe('object storage end-to-end tests', () => { */ it('can create and delete object storage buckets', () => { cy.tag('purpose:syntheticTesting'); - throw new Error('Fail'); const bucketLabel = randomLabel(); const bucketRegion = 'Atlanta, GA'; From 90170088518459157313a5955c62a5ff26c28ae7 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 25 Jul 2024 11:54:17 -0400 Subject: [PATCH 08/12] Allow retries to be disabled in CI via environment variable --- docs/development-guide/08-testing.md | 1 + packages/manager/cypress.config.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/development-guide/08-testing.md b/docs/development-guide/08-testing.md index febe5b53519..48fcf586379 100644 --- a/docs/development-guide/08-testing.md +++ b/docs/development-guide/08-testing.md @@ -215,6 +215,7 @@ Environment variables related to Cypress logging and reporting, as well as repor | `CY_TEST_USER_REPORT` | Log test account information when tests begin | `1` | Unset; disabled by default | | `CY_TEST_JUNIT_REPORT` | Enable JUnit reporting | `1` | Unset; disabled by default | | `CY_TEST_DISABLE_FILE_WATCHING` | Disable file watching in Cypress UI | `1` | Unset; disabled by default | +| `CY_TEST_DISABLE_RETRIES` | Disable test retries on failure in CI | `1` | Unset; disabled by default | | `CY_TEST_FAIL_ON_MANAGED` | Fail affected tests when Managed is enabled | `1` | Unset; disabled by default | ### Writing End-to-End Tests diff --git a/packages/manager/cypress.config.ts b/packages/manager/cypress.config.ts index 1c7efb41af8..c0cc12e20a3 100644 --- a/packages/manager/cypress.config.ts +++ b/packages/manager/cypress.config.ts @@ -41,7 +41,7 @@ export default defineConfig({ video: true, // Only retry test when running via CI. - retries: process.env['CI'] ? 2 : 0, + retries: process.env['CI'] && !process.env['CY_TEST_DISABLE_RETRIES'] ? 2 : 0, experimentalMemoryManagement: true, e2e: { From 94c48d2cf812f2f191983d9b7f3b3637ccd81736 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 25 Jul 2024 11:56:15 -0400 Subject: [PATCH 09/12] Pass new environment variable to Docker --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 41d4c594fe4..51f2f01e6a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,7 @@ x-e2e-env: CY_TEST_SUITE: ${CY_TEST_SUITE} CY_TEST_REGION: ${CY_TEST_REGION} CY_TEST_TAGS: ${CY_TEST_TAGS} + CY_TEST_DISABLE_RETRIES: ${CY_TEST_DISABLE_RETRIES} # Cypress environment variables for alternative parallelization. CY_TEST_SPLIT_RUN: ${CY_TEST_SPLIT_RUN} From 4fd50c51b947274bd52f9521fc93d92003ac10b7 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 25 Jul 2024 11:58:05 -0400 Subject: [PATCH 10/12] Temporarily trigger test failure again --- .../cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts index 3c6b63c6ec0..28ae5b6e795 100644 --- a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts +++ b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts @@ -157,6 +157,7 @@ describe('object storage end-to-end tests', () => { */ it('can create and delete object storage buckets', () => { cy.tag('purpose:syntheticTesting'); + throw new Error('This failure was triggered intentionally.'); const bucketLabel = randomLabel(); const bucketRegion = 'Atlanta, GA'; From ea72c2cdb8c6e63ceac4a88ebe635f1c2f63122f Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Thu, 25 Jul 2024 12:02:52 -0400 Subject: [PATCH 11/12] Stop test from failing --- .../cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts index 28ae5b6e795..3c6b63c6ec0 100644 --- a/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts +++ b/packages/manager/cypress/e2e/core/objectStorage/object-storage.e2e.spec.ts @@ -157,7 +157,6 @@ describe('object storage end-to-end tests', () => { */ it('can create and delete object storage buckets', () => { cy.tag('purpose:syntheticTesting'); - throw new Error('This failure was triggered intentionally.'); const bucketLabel = randomLabel(); const bucketRegion = 'Atlanta, GA'; From 85545f628d857eda05a479810101ba2537f8f5f9 Mon Sep 17 00:00:00 2001 From: Joe D'Amore Date: Fri, 26 Jul 2024 14:30:02 -0400 Subject: [PATCH 12/12] Add changesets --- .../.changeset/pr-10713-tech-stories-1722017812325.md | 5 +++++ packages/manager/.changeset/pr-10713-tests-1722017832242.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 packages/manager/.changeset/pr-10713-tech-stories-1722017812325.md create mode 100644 packages/manager/.changeset/pr-10713-tests-1722017832242.md diff --git a/packages/manager/.changeset/pr-10713-tech-stories-1722017812325.md b/packages/manager/.changeset/pr-10713-tech-stories-1722017812325.md new file mode 100644 index 00000000000..6e5dd981cb2 --- /dev/null +++ b/packages/manager/.changeset/pr-10713-tech-stories-1722017812325.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tech Stories +--- + +Docker Compose changes to facilitate new testing pipeline ([#10713](https://github.com/linode/manager/pull/10713)) diff --git a/packages/manager/.changeset/pr-10713-tests-1722017832242.md b/packages/manager/.changeset/pr-10713-tests-1722017832242.md new file mode 100644 index 00000000000..4e2dbac52ad --- /dev/null +++ b/packages/manager/.changeset/pr-10713-tests-1722017832242.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Tag tests for synthetic monitoring ([#10713](https://github.com/linode/manager/pull/10713))