Merge pull request #3016 from cdr/jsjoeio/refactor-e2e

dev(testing): add jest-playwright and reduce flakiness of e2e tests

Author: jsjoeio

.eslintrc.yaml

@@ -15,7 +15,12 @@ extends:
   - plugin:import/recommended
   - plugin:import/typescript
   - plugin:prettier/recommended
-  - prettier # Removes eslint rules that conflict with prettier.
+  # Recommended by jest-playwright
+  # https://github.com/playwright-community/jest-playwright#globals
+  - plugin:jest-playwright/recommended
+  # Prettier should always be last
+  # Removes eslint rules that conflict with prettier.
+  - prettier
 
 rules:
   # Sometimes you need to add args to implement a function signature even

.github/workflows/ci.yaml

@@ -63,7 +63,7 @@ jobs:
       - uses: microsoft/playwright-github-action@v1
       - name: Install dependencies and run end-to-end tests
         run: |
-          ./release-packages/code-server*-linux-amd64/bin/code-server &
+          ./release-packages/code-server*-linux-amd64/bin/code-server --log trace &
           yarn --frozen-lockfile
           yarn test:e2e
       - name: Upload test artifacts

ci/dev/test-e2e.sh

@@ -15,7 +15,7 @@ main() {
     echo -e "\n"
     exit 1
   fi
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts --runInBand
 }
 
 main "$@"

ci/steps/test-e2e.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 main() {
   cd "$(dirname "$0")/../.."
 
-  "./release-packages/code-server*-linux-amd64/bin/code-server" --home "$CODE_SERVER_ADDRESS"/healthz &
+  "./release-packages/code-server*-linux-amd64/bin/code-server" &
   yarn --frozen-lockfile
   yarn test:e2e
 }

lib/vscode/src/vs/platform/extensionManagement/node/extensionDownloader.ts

@@ -76,7 +76,7 @@ export class ExtensionsDownloader extends Disposable {
 	private async cleanUp(): Promise<void> {
 		try {
 			if (!(await this.fileService.exists(this.extensionsDownloadDir))) {
-				this.logService.trace('Extension VSIX downlads cache dir does not exist');
+				this.logService.trace('Extension VSIX downloads cache dir does not exist');
 				return;
 			}
 			const folderStat = await this.fileService.resolve(this.extensionsDownloadDir, { resolveMetadata: true });

package.json

@@ -61,6 +61,7 @@
     "eslint-config-prettier": "^8.1.0",
     "eslint-import-resolver-alias": "^1.1.2",
     "eslint-plugin-import": "^2.18.2",
+    "eslint-plugin-jest-playwright": "^0.2.1",
     "eslint-plugin-prettier": "^3.1.0",
     "istanbul-badges-readme": "^1.2.0",
     "leaked-handles": "^5.2.0",

test/e2e/browser.test.ts

@@ -0,0 +1,35 @@
+/// <reference types="jest-playwright-preset" />
+
+// This test is for nothing more than to make sure
+// tests are running in multiple browsers
+describe("Browser gutcheck", () => {
+  beforeEach(async () => {
+    await jestPlaywright.resetBrowser({
+      logger: {
+        isEnabled: (name) => name === "browser",
+        log: (name, severity, message, args) => console.log(`${name} ${message}`),
+      },
+    })
+  })
+
+  test("should display correct browser based on userAgent", async () => {
+    const displayNames = {
+      chromium: "Chrome",
+      firefox: "Firefox",
+      webkit: "Safari",
+    }
+    const userAgent = await page.evaluate("navigator.userAgent")
+
+    if (browserName === "chromium") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+
+    if (browserName === "firefox") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+
+    if (browserName === "webkit") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+  })
+})

test/e2e/e2e.test.ts

@@ -0,0 +1,21 @@
+/// <reference types="jest-playwright-preset" />
+import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+
+// This test is to make sure the globalSetup works as expected
+// meaning globalSetup ran and stored the storageState in STORAGE
+describe("globalSetup", () => {
+  beforeEach(async () => {
+    // Create a new context with the saved storage state
+    // so we don't have to logged in
+    const storageState = JSON.parse(STORAGE) || {}
+    await jestPlaywright.resetContext({
+      storageState,
+    })
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+  })
+
+  it("should keep us logged in using the storageState", async () => {
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
+  })
+})

test/e2e/globalSetup.test.ts

@@ -1,38 +1,19 @@
-import { chromium, Page, Browser, BrowserContext } from "playwright"
+/// <reference types="jest-playwright-preset" />
 import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
 describe("login", () => {
-  let browser: Browser
-  let page: Page
-  let context: BrowserContext
-
-  beforeAll(async () => {
-    browser = await chromium.launch()
-    context = await browser.newContext()
-  })
-
-  afterAll(async () => {
-    await browser.close()
-  })
-
   beforeEach(async () => {
-    page = await context.newPage()
-  })
-
-  afterEach(async () => {
-    await page.close()
-    // Remove password from local storage
-    await context.clearCookies()
+    await jestPlaywright.resetBrowser()
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   })
 
   it("should be able to login", async () => {
-    await page.goto(CODE_SERVER_ADDRESS)
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
-    // See the editor
-    const codeServerEditor = await page.isVisible(".monaco-workbench")
-    expect(codeServerEditor).toBeTruthy()
+    await page.waitForLoadState("networkidle")
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
   })
 })