run tests 100x to check flakiness

getsentry · Mar 3, 2023 · 53dd25a · 53dd25a
1 parent c6135dd
commit 53dd25a
Show file tree

Hide file tree

Showing 4 changed files with 191 additions and 181 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -481,7 +481,7 @@ jobs:
     needs: [job_get_metadata, job_build]
     if: needs.job_get_metadata.outputs.changed_browser_integration == 'true' || github.event_name != 'pull_request'
     runs-on: ubuntu-20.04
-    timeout-minutes: 15
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       matrix:

diff --git a/packages/integration-tests/suites/replay/sessionExpiry/test.ts b/packages/integration-tests/suites/replay/sessionExpiry/test.ts
@@ -15,80 +15,84 @@ import {
 // Session should expire after 2s - keep in sync with init.js
 const SESSION_TIMEOUT = 2000;
 
-sentryTest('handles an expired session', async ({ getLocalTestPath, page }) => {
-  if (shouldSkipReplayTest()) {
-    sentryTest.skip();
-  }
-
-  const reqPromise0 = waitForReplayRequest(page, 0);
-  const reqPromise1 = waitForReplayRequest(page, 1);
-
-  await page.route('https://dsn.ingest.sentry.io/**/*', route => {
-    return route.fulfill({
-      status: 200,
-      contentType: 'application/json',
-      body: JSON.stringify({ id: 'test-id' }),
+for (let i = 0; i < 100; i++) {
+  sentryTest(`handles an expired session RUN ${i}`, async ({ getLocalTestPath, page }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    const reqPromise0 = waitForReplayRequest(page, 0);
+    const reqPromise1 = waitForReplayRequest(page, 1);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
     });
-  });
 
-  const url = await getLocalTestPath({ testDir: __dirname });
+    const url = await getLocalTestPath({ testDir: __dirname });
 
-  await page.goto(url);
+    await page.goto(url);
 
-  const replayEvent0 = getReplayEvent(await reqPromise0);
-  expect(replayEvent0).toEqual(getExpectedReplayEvent({}));
+    const replayEvent0 = getReplayEvent(await reqPromise0);
+    expect(replayEvent0).toEqual(getExpectedReplayEvent({}));
 
-  const fullSnapshots0 = getFullRecordingSnapshots(await reqPromise0);
-  expect(fullSnapshots0.length).toEqual(1);
-  const stringifiedSnapshot = normalize(fullSnapshots0[0]);
-  expect(stringifiedSnapshot).toMatchSnapshot('snapshot-0.json');
+    const fullSnapshots0 = getFullRecordingSnapshots(await reqPromise0);
+    expect(fullSnapshots0.length).toEqual(1);
+    const stringifiedSnapshot = normalize(fullSnapshots0[0]);
+    expect(stringifiedSnapshot).toMatchSnapshot('snapshot-0.json');
 
-  // We wait for another segment 0
-  const reqPromise2 = waitForReplayRequest(page, 0);
+    // We wait for another segment 0
+    const reqPromise2 = waitForReplayRequest(page, 0);
 
-  await page.click('#button1');
-  const replayEvent1 = getReplayEvent(await reqPromise1);
-  expect(replayEvent1).toEqual(getExpectedReplayEvent({ replay_start_timestamp: undefined, segment_id: 1, urls: [] }));
+    await page.click('#button1');
+    const replayEvent1 = getReplayEvent(await reqPromise1);
+    expect(replayEvent1).toEqual(
+      getExpectedReplayEvent({ replay_start_timestamp: undefined, segment_id: 1, urls: [] }),
+    );
 
-  const fullSnapshots1 = getFullRecordingSnapshots(await reqPromise1);
-  expect(fullSnapshots1.length).toEqual(0);
+    const fullSnapshots1 = getFullRecordingSnapshots(await reqPromise1);
+    expect(fullSnapshots1.length).toEqual(0);
 
-  const incrementalSnapshots1 = getIncrementalRecordingSnapshots(await reqPromise1);
-  // The number of incremental snapshots depends on the browser
-  expect(incrementalSnapshots1.length).toBeGreaterThanOrEqual(4);
+    const incrementalSnapshots1 = getIncrementalRecordingSnapshots(await reqPromise1);
+    // The number of incremental snapshots depends on the browser
+    expect(incrementalSnapshots1.length).toBeGreaterThanOrEqual(4);
 
-  expect(incrementalSnapshots1).toEqual(
-    expect.arrayContaining([
-      {
-        source: 1,
-        positions: [
-          {
-            id: 9,
-            timeOffset: expect.any(Number),
-            x: expect.any(Number),
-            y: expect.any(Number),
-          },
-        ],
-      },
-    ]),
-  );
+    expect(incrementalSnapshots1).toEqual(
+      expect.arrayContaining([
+        {
+          source: 1,
+          positions: [
+            {
+              id: 9,
+              timeOffset: expect.any(Number),
+              x: expect.any(Number),
+              y: expect.any(Number),
+            },
+          ],
+        },
+      ]),
+    );
 
-  const replay = await getReplaySnapshot(page);
-  const oldSessionId = replay.session?.id;
+    const replay = await getReplaySnapshot(page);
+    const oldSessionId = replay.session?.id;
 
-  await new Promise(resolve => setTimeout(resolve, SESSION_TIMEOUT));
+    await new Promise(resolve => setTimeout(resolve, SESSION_TIMEOUT));
 
-  await page.click('#button2');
+    await page.click('#button2');
 
-  const replay2 = await getReplaySnapshot(page);
+    const replay2 = await getReplaySnapshot(page);
 
-  expect(replay2.session?.id).not.toEqual(oldSessionId);
+    expect(replay2.session?.id).not.toEqual(oldSessionId);
 
-  const replayEvent2 = getReplayEvent(await reqPromise2);
-  expect(replayEvent2).toEqual(getExpectedReplayEvent({}));
+    const replayEvent2 = getReplayEvent(await reqPromise2);
+    expect(replayEvent2).toEqual(getExpectedReplayEvent({}));
 
-  const fullSnapshots2 = getFullRecordingSnapshots(await reqPromise2);
-  expect(fullSnapshots2.length).toEqual(1);
-  const stringifiedSnapshot2 = normalize(fullSnapshots2[0]);
-  expect(stringifiedSnapshot2).toMatchSnapshot('snapshot-2.json');
-});
+    const fullSnapshots2 = getFullRecordingSnapshots(await reqPromise2);
+    expect(fullSnapshots2.length).toEqual(1);
+    const stringifiedSnapshot2 = normalize(fullSnapshots2[0]);
+    expect(stringifiedSnapshot2).toMatchSnapshot('snapshot-2.json');
+  });
+}
diff --git a/packages/integration-tests/suites/replay/sessionInactive/test.ts b/packages/integration-tests/suites/replay/sessionInactive/test.ts
@@ -14,73 +14,75 @@ import {
 // Session should expire after 2s - keep in sync with init.js
 const SESSION_TIMEOUT = 2000;
 
-sentryTest('handles an inactive session', async ({ getLocalTestPath, page }) => {
-  if (shouldSkipReplayTest()) {
-    sentryTest.skip();
-  }
-
-  const reqPromise0 = waitForReplayRequest(page, 0);
-
-  await page.route('https://dsn.ingest.sentry.io/**/*', route => {
-    return route.fulfill({
-      status: 200,
-      contentType: 'application/json',
-      body: JSON.stringify({ id: 'test-id' }),
+for (let i = 0; i < 100; i++) {
+  sentryTest(`handles an inactive session RUN ${i}`, async ({ getLocalTestPath, page }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    const reqPromise0 = waitForReplayRequest(page, 0);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
     });
-  });
 
-  const url = await getLocalTestPath({ testDir: __dirname });
+    const url = await getLocalTestPath({ testDir: __dirname });
 
-  await page.goto(url);
+    await page.goto(url);
 
-  const replayEvent0 = getReplayEvent(await reqPromise0);
-  expect(replayEvent0).toEqual(getExpectedReplayEvent({}));
+    const replayEvent0 = getReplayEvent(await reqPromise0);
+    expect(replayEvent0).toEqual(getExpectedReplayEvent({}));
 
-  const fullSnapshots0 = getFullRecordingSnapshots(await reqPromise0);
-  expect(fullSnapshots0.length).toEqual(1);
-  const stringifiedSnapshot = normalize(fullSnapshots0[0]);
-  expect(stringifiedSnapshot).toMatchSnapshot('snapshot-0.json');
+    const fullSnapshots0 = getFullRecordingSnapshots(await reqPromise0);
+    expect(fullSnapshots0.length).toEqual(1);
+    const stringifiedSnapshot = normalize(fullSnapshots0[0]);
+    expect(stringifiedSnapshot).toMatchSnapshot('snapshot-0.json');
 
-  await page.click('#button1');
+    await page.click('#button1');
 
-  // We wait for another segment 0
-  const reqPromise1 = waitForReplayRequest(page, 0);
+    // We wait for another segment 0
+    const reqPromise1 = waitForReplayRequest(page, 0);
 
-  // Now we wait for the session timeout, nothing should be sent in the meanwhile
-  await new Promise(resolve => setTimeout(resolve, SESSION_TIMEOUT));
+    // Now we wait for the session timeout, nothing should be sent in the meanwhile
+    await new Promise(resolve => setTimeout(resolve, SESSION_TIMEOUT));
 
-  // nothing happened because no activity/inactivity was detected
-  const replay = await getReplaySnapshot(page);
-  // @ts-ignore private api
-  expect(replay._isEnabled).toEqual(true);
-  // @ts-ignore private api
-  expect(replay._isPaused).toEqual(false);
+    // nothing happened because no activity/inactivity was detected
+    const replay = await getReplaySnapshot(page);
+    // @ts-ignore private api
+    expect(replay._isEnabled).toEqual(true);
+    // @ts-ignore private api
+    expect(replay._isPaused).toEqual(false);
 
-  // Now we trigger a blur event, which should move the session to paused mode
-  await page.evaluate(() => {
-    window.dispatchEvent(new Event('blur'));
-  });
+    // Now we trigger a blur event, which should move the session to paused mode
+    await page.evaluate(() => {
+      window.dispatchEvent(new Event('blur'));
+    });
+
+    const replay2 = await getReplaySnapshot(page);
+    // @ts-ignore private api
+    expect(replay2._isEnabled).toEqual(true);
+    // @ts-ignore private api
+    expect(replay2._isPaused).toEqual(true);
 
-  const replay2 = await getReplaySnapshot(page);
-  // @ts-ignore private api
-  expect(replay2._isEnabled).toEqual(true);
-  // @ts-ignore private api
-  expect(replay2._isPaused).toEqual(true);
-
-  // Trigger an action, should re-start the recording
-  await page.click('#button2');
-
-  const replay3 = await getReplaySnapshot(page);
-  // @ts-ignore private api
-  expect(replay3._isEnabled).toEqual(true);
-  // @ts-ignore private api
-  expect(replay3._isPaused).toEqual(false);
-
-  const replayEvent1 = getReplayEvent(await reqPromise1);
-  expect(replayEvent1).toEqual(getExpectedReplayEvent({}));
-
-  const fullSnapshots1 = getFullRecordingSnapshots(await reqPromise0);
-  expect(fullSnapshots1.length).toEqual(1);
-  const stringifiedSnapshot1 = normalize(fullSnapshots1[0]);
-  expect(stringifiedSnapshot1).toMatchSnapshot('snapshot-1.json');
-});
+    // Trigger an action, should re-start the recording
+    await page.click('#button2');
+
+    const replay3 = await getReplaySnapshot(page);
+    // @ts-ignore private api
+    expect(replay3._isEnabled).toEqual(true);
+    // @ts-ignore private api
+    expect(replay3._isPaused).toEqual(false);
+
+    const replayEvent1 = getReplayEvent(await reqPromise1);
+    expect(replayEvent1).toEqual(getExpectedReplayEvent({}));
+
+    const fullSnapshots1 = getFullRecordingSnapshots(await reqPromise0);
+    expect(fullSnapshots1.length).toEqual(1);
+    const stringifiedSnapshot1 = normalize(fullSnapshots1[0]);
+    expect(stringifiedSnapshot1).toMatchSnapshot('snapshot-1.json');
+  });
+}