From 6e55f7dbf5a31afe1d1f3fab050bfb72ba199bc8 Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Fri, 19 Jan 2024 00:09:06 +0000 Subject: [PATCH 1/8] fix(tests): Fix flaky Remix e2e tests. --- .github/workflows/build.yml | 4 ++-- .../create-remix-app-v2/tests/behaviour-client.test.ts | 6 ++---- .../create-remix-app/tests/behaviour-client.test.ts | 6 ++---- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3856ea843d7e..54a954b633cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1043,8 +1043,8 @@ jobs: 'create-react-app', 'create-next-app', # disabling remix e2e tests because of flakes - # 'create-remix-app', - # 'create-remix-app-v2', + 'create-remix-app', + 'create-remix-app-v2', 'debug-id-sourcemaps', 'nextjs-app-dir', 'nextjs-14', diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts index 3427ba3b679e..944dcb07b4bd 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts @@ -47,8 +47,7 @@ test('Sends a client-side exception to Sentry', async ({ page }) => { .toBe(200); }); -// Skipping test because of flake -test.skip('Sends a pageload transaction to Sentry', async ({ page }) => { +test('Sends a pageload transaction to Sentry', async ({ page }) => { await page.goto('/'); const recordedTransactionsHandle = await page.waitForFunction(() => { @@ -107,8 +106,7 @@ test.skip('Sends a pageload transaction to Sentry', async ({ page }) => { expect(hadPageLoadTransaction).toBe(true); }); -// Skipped because of test flake -test.skip('Sends a navigation transaction to Sentry', async ({ page }) => { +test('Sends a navigation transaction to Sentry', async ({ page }) => { await page.goto('/'); // Give pageload transaction time to finish diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts b/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts index f760d9c5a629..944dcb07b4bd 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts @@ -7,8 +7,7 @@ const authToken = process.env.E2E_TEST_AUTH_TOKEN; const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG; const sentryTestProject = process.env.E2E_TEST_SENTRY_TEST_PROJECT; -// skipping flaky test -test.skip('Sends a client-side exception to Sentry', async ({ page }) => { +test('Sends a client-side exception to Sentry', async ({ page }) => { await page.goto('/'); const exceptionButton = page.locator('id=exception-button'); @@ -48,8 +47,7 @@ test.skip('Sends a client-side exception to Sentry', async ({ page }) => { .toBe(200); }); -// skipping flaky test -test.skip('Sends a pageload transaction to Sentry', async ({ page }) => { +test('Sends a pageload transaction to Sentry', async ({ page }) => { await page.goto('/'); const recordedTransactionsHandle = await page.waitForFunction(() => { From 7adb4cbfa79138025d0b809a297a4f88dc4b9bc7 Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Fri, 19 Jan 2024 10:18:11 +0000 Subject: [PATCH 2/8] Turn on debug mode. --- .../test-applications/create-remix-app-v2/app/entry.client.tsx | 1 + .../test-applications/create-remix-app-v2/app/entry.server.tsx | 1 + .../test-applications/create-remix-app/app/entry.client.tsx | 1 + .../test-applications/create-remix-app/app/entry.server.tsx | 1 + 4 files changed, 4 insertions(+) diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx index 15cb6513a917..f2020cc15591 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx @@ -22,6 +22,7 @@ Sentry.init({ // Session Replay replaysSessionSampleRate: 0.1, // This sets the sample rate at 10%. You may want to change it to 100% while in development and then sample at a lower rate in production. replaysOnErrorSampleRate: 1.0, // If you're not already sampling the entire session, change the sample rate to 100% when sampling sessions where errors occur. + debug: true, }); Sentry.addEventProcessor(event => { diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx index c2e0a480acab..877ea175b71b 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx @@ -22,6 +22,7 @@ Sentry.init({ dsn: process.env.E2E_TEST_DSN, // Performance Monitoring tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production! + debug: true, }); export const handleError = Sentry.wrapRemixHandleError; diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx index 15cb6513a917..f2020cc15591 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx @@ -22,6 +22,7 @@ Sentry.init({ // Session Replay replaysSessionSampleRate: 0.1, // This sets the sample rate at 10%. You may want to change it to 100% while in development and then sample at a lower rate in production. replaysOnErrorSampleRate: 1.0, // If you're not already sampling the entire session, change the sample rate to 100% when sampling sessions where errors occur. + debug: true, }); Sentry.addEventProcessor(event => { diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx index 6d137ec92d53..45cf290dcca0 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx @@ -19,6 +19,7 @@ Sentry.init({ dsn: process.env.E2E_TEST_DSN, // Performance Monitoring tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production! + debug: true, }); export const handleError = Sentry.wrapRemixHandleError; From b267d84142b7a67404948afa8d7095c807e605a2 Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Mon, 22 Jan 2024 17:22:28 +0000 Subject: [PATCH 3/8] Turn off debug mode. --- .../test-applications/create-remix-app-v2/app/entry.client.tsx | 1 - .../test-applications/create-remix-app-v2/app/entry.server.tsx | 1 - .../test-applications/create-remix-app/app/entry.client.tsx | 1 - .../test-applications/create-remix-app/app/entry.server.tsx | 1 - 4 files changed, 4 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx index f2020cc15591..15cb6513a917 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx @@ -22,7 +22,6 @@ Sentry.init({ // Session Replay replaysSessionSampleRate: 0.1, // This sets the sample rate at 10%. You may want to change it to 100% while in development and then sample at a lower rate in production. replaysOnErrorSampleRate: 1.0, // If you're not already sampling the entire session, change the sample rate to 100% when sampling sessions where errors occur. - debug: true, }); Sentry.addEventProcessor(event => { diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx index 877ea175b71b..c2e0a480acab 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx @@ -22,7 +22,6 @@ Sentry.init({ dsn: process.env.E2E_TEST_DSN, // Performance Monitoring tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production! - debug: true, }); export const handleError = Sentry.wrapRemixHandleError; diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx index f2020cc15591..15cb6513a917 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx @@ -22,7 +22,6 @@ Sentry.init({ // Session Replay replaysSessionSampleRate: 0.1, // This sets the sample rate at 10%. You may want to change it to 100% while in development and then sample at a lower rate in production. replaysOnErrorSampleRate: 1.0, // If you're not already sampling the entire session, change the sample rate to 100% when sampling sessions where errors occur. - debug: true, }); Sentry.addEventProcessor(event => { diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx index 45cf290dcca0..6d137ec92d53 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx @@ -19,7 +19,6 @@ Sentry.init({ dsn: process.env.E2E_TEST_DSN, // Performance Monitoring tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production! - debug: true, }); export const handleError = Sentry.wrapRemixHandleError; From ae17485485c30443992a3e4b33c30ca3dbfb255a Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Mon, 22 Jan 2024 19:13:41 +0000 Subject: [PATCH 4/8] Delete comment --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54a954b633cd..14d255bae1a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1042,7 +1042,6 @@ jobs: 'node-express-app', 'create-react-app', 'create-next-app', - # disabling remix e2e tests because of flakes 'create-remix-app', 'create-remix-app-v2', 'debug-id-sourcemaps', From fc18aed207889bf1bc6077442b5b335a19273ca6 Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Sun, 4 Feb 2024 17:18:34 +0000 Subject: [PATCH 5/8] Increase polling timeout to 120 seconds. --- .../create-remix-app-v2/tests/behaviour-client.test.ts | 2 +- .../create-remix-app/tests/behaviour-client.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts index 944dcb07b4bd..6cdfa651a25d 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts @@ -1,7 +1,7 @@ import { expect, test } from '@playwright/test'; import axios, { AxiosError } from 'axios'; -const EVENT_POLLING_TIMEOUT = 90_000; +const EVENT_POLLING_TIMEOUT = 120_000; const authToken = process.env.E2E_TEST_AUTH_TOKEN; const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG; diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts b/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts index 944dcb07b4bd..6cdfa651a25d 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts @@ -1,7 +1,7 @@ import { expect, test } from '@playwright/test'; import axios, { AxiosError } from 'axios'; -const EVENT_POLLING_TIMEOUT = 90_000; +const EVENT_POLLING_TIMEOUT = 120_000; const authToken = process.env.E2E_TEST_AUTH_TOKEN; const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG; From 01f4c0d0af731168ab853d6d4e206343d9c28e78 Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Mon, 12 Feb 2024 15:25:50 +0000 Subject: [PATCH 6/8] Remove unused `entry` in remix test config. --- .../test-applications/create-remix-app-v2/remix.config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/remix.config.js b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/remix.config.js index 92ed1ddc32eb..cb3c8c7a9fb7 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/remix.config.js +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/remix.config.js @@ -6,5 +6,4 @@ module.exports = { // serverBuildPath: 'build/index.js', // publicPath: '/build/', serverModuleFormat: 'cjs', - entry, }; From cf378a43d4371dbbbcf5c0b177eb485932666e9e Mon Sep 17 00:00:00 2001 From: Onur Temizkan Date: Mon, 12 Feb 2024 20:17:54 +0000 Subject: [PATCH 7/8] Set `qa` as the environment to bypass dynamic sampling. --- .../test-applications/create-remix-app-v2/app/entry.client.tsx | 1 + .../test-applications/create-remix-app-v2/app/entry.server.tsx | 1 + .../test-applications/create-remix-app/app/entry.client.tsx | 1 + .../test-applications/create-remix-app/app/entry.server.tsx | 1 + 4 files changed, 4 insertions(+) diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx index 15cb6513a917..a56d13dddadc 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.client.tsx @@ -10,6 +10,7 @@ import { StrictMode, startTransition, useEffect } from 'react'; import { hydrateRoot } from 'react-dom/client'; Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions dsn: window.ENV.SENTRY_DSN, integrations: [ new Sentry.BrowserTracing({ diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx index c2e0a480acab..d8e63095fa43 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/app/entry.server.tsx @@ -19,6 +19,7 @@ installGlobals(); const ABORT_DELAY = 5_000; Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions dsn: process.env.E2E_TEST_DSN, // Performance Monitoring tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production! diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx index 15cb6513a917..a56d13dddadc 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx @@ -10,6 +10,7 @@ import { StrictMode, startTransition, useEffect } from 'react'; import { hydrateRoot } from 'react-dom/client'; Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions dsn: window.ENV.SENTRY_DSN, integrations: [ new Sentry.BrowserTracing({ diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx index 6d137ec92d53..c2de73cdba63 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx @@ -16,6 +16,7 @@ import { renderToPipeableStream } from 'react-dom/server'; const ABORT_DELAY = 5_000; Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions dsn: process.env.E2E_TEST_DSN, // Performance Monitoring tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production! From 8658d525ed2a4b59c6e5506a00efe3aaadf3af4a Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 14 Feb 2024 10:32:26 +0000 Subject: [PATCH 8/8] Reduce timeout to 90 seconds --- .../create-remix-app-v2/tests/behaviour-client.test.ts | 2 +- .../create-remix-app/tests/behaviour-client.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts index 6cdfa651a25d..944dcb07b4bd 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts +++ b/dev-packages/e2e-tests/test-applications/create-remix-app-v2/tests/behaviour-client.test.ts @@ -1,7 +1,7 @@ import { expect, test } from '@playwright/test'; import axios, { AxiosError } from 'axios'; -const EVENT_POLLING_TIMEOUT = 120_000; +const EVENT_POLLING_TIMEOUT = 90_000; const authToken = process.env.E2E_TEST_AUTH_TOKEN; const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG; diff --git a/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts b/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts index 6cdfa651a25d..944dcb07b4bd 100644 --- a/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts +++ b/dev-packages/e2e-tests/test-applications/create-remix-app/tests/behaviour-client.test.ts @@ -1,7 +1,7 @@ import { expect, test } from '@playwright/test'; import axios, { AxiosError } from 'axios'; -const EVENT_POLLING_TIMEOUT = 120_000; +const EVENT_POLLING_TIMEOUT = 90_000; const authToken = process.env.E2E_TEST_AUTH_TOKEN; const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG;