Fix a set of E2E bugs causing several CI failures (#2177)

* Fix E2E bugs causing CI failutes

* Revert setup in dx

Co-authored-by: zomars <zomars@me.com>

.github/workflows/e2e.yml

@@ -13,7 +13,6 @@ jobs:
       DATABASE_URL: postgresql://postgres:@localhost:5432/calendso
       BASE_URL: http://localhost:3000
       JWT_SECRET: secret
-      PLAYWRIGHT_SECRET: ${{ secrets.CI_PLAYWRIGHT_SECRET }}
       GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }}
       GOOGLE_LOGIN_ENABLED: true
       # CRON_API_KEY: xxx

README.md

@@ -196,7 +196,7 @@ echo 'NEXT_PUBLIC_DEBUG=1' >> apps/web/.env
 ### E2E-Testing
 
 ```sh
-# In first terminal
+# In first terminal. Must run on port 3000.
 yarn dx
 # In second terminal
 yarn workspace @calcom/web test-e2e

apps/web/.env.example

@@ -25,7 +25,6 @@ NEXT_PUBLIC_APP_URL='http://localhost:3000'
 
 JWT_SECRET='secret'
 # This is used so we can bypass emails in auth flows for E2E testing
-PLAYWRIGHT_SECRET=
 
 # To enable SAML login, set both these variables
 # @see https://github.com/calcom/cal.com/tree/main/packages/ee#setting-up-saml-login
@@ -101,4 +100,7 @@ CALENDSO_ENCRYPTION_KEY=
 NEXT_PUBLIC_INTERCOM_APP_ID=
 
 # Zendesk Config
-NEXT_PUBLIC_ZENDESK_KEY=
+NEXT_PUBLIC_ZENDESK_KEY=
+
+# Set it to "1" if you need to run E2E tests locally
+NEXT_PUBLIC_IS_E2E=

apps/web/pages/api/auth/forgot-password.ts

@@ -65,10 +65,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     await sendPasswordResetEmail(passwordEmail);
 
     /** So we can test the password reset flow on CI */
-    if (
-      process.env.PLAYWRIGHT_SECRET &&
-      req.headers["x-playwright-secret"] === process.env.PLAYWRIGHT_SECRET
-    ) {
+    if (process.env.NEXT_PUBLIC_IS_E2E) {
       return res.status(201).json({ message: "Reset Requested", resetLink });
     }
 

apps/web/playwright/auth/forgot-password.test.ts

@@ -1,17 +1,6 @@
 import { expect, test } from "@playwright/test";
 
-test("Can reset forgotten password", async ({ browser }) => {
-  test.fixme(true, "TODO: This test is failing randomly, disabled for now");
-  // Create a new incognito browser context
-  const context = await browser.newContext({
-    extraHTTPHeaders: {
-      // Only needed for bypassing emails while testing
-      "X-Playwright-Secret": process.env.PLAYWRIGHT_SECRET || "",
-    },
-  });
-  // Create a new page inside context.
-  const page = await context.newPage();
-
+test("Can reset forgotten password", async ({ page }) => {
   // Got to reset password flow
   await page.goto("/auth/forgot-password");
 
@@ -50,6 +39,4 @@ test("Can reset forgotten password", async ({ browser }) => {
   await page.waitForSelector("[data-testid=dashboard-shell]");
 
   await expect(page.locator("[data-testid=dashboard-shell]")).toBeVisible();
-
-  await context.close();
 });

apps/web/playwright/booking-pages.test.ts

@@ -1,18 +1,8 @@
 import { expect, test } from "@playwright/test";
 
-import prisma from "@lib/prisma";
-
+import { deleteAllBookingsByEmail } from "./lib/teardown";
 import { selectFirstAvailableTimeSlotNextMonth, todo } from "./lib/testUtils";
 
-const deleteBookingsByEmail = async (email: string) =>
-  prisma.booking.deleteMany({
-    where: {
-      user: {
-        email,
-      },
-    },
-  });
-
 test.describe("free user", () => {
   test.beforeEach(async ({ page }) => {
     await page.goto("/free");
@@ -20,7 +10,7 @@ test.describe("free user", () => {
 
   test.afterEach(async () => {
     // delete test bookings
-    await deleteBookingsByEmail("free@example.com");
+    await deleteAllBookingsByEmail("free@example.com");
   });
 
   test("only one visible event", async ({ page }) => {
@@ -81,14 +71,15 @@ test.describe("pro user", () => {
     await page.goto("/pro");
   });
 
-  test.afterEach(async () => {
+  test.afterAll(async () => {
     // delete test bookings
-    await deleteBookingsByEmail("pro@example.com");
+    await deleteAllBookingsByEmail("pro@example.com");
   });
 
   test("pro user's page has at least 2 visible events", async ({ page }) => {
-    const $eventTypes = await page.$$("[data-testid=event-types] > *");
-    expect($eventTypes.length).toBeGreaterThanOrEqual(2);
+    // await page.pause();
+    const $eventTypes = await page.locator("[data-testid=event-types] > *");
+    expect(await $eventTypes.count()).toBeGreaterThanOrEqual(2);
   });
 
   test("book an event first day in next month", async ({ page }) => {

apps/web/playwright/change-password.test.ts

@@ -1,28 +1,30 @@
 import { expect, test } from "@playwright/test";
 
-// Using logged in state from globalSteup
-test.use({ storageState: "playwright/artifacts/proStorageState.json" });
+test.describe("Change Passsword Test", () => {
+  // Using logged in state from globalSteup
+  test.use({ storageState: "playwright/artifacts/proStorageState.json" });
 
-test("change password", async ({ page }) => {
-  // Try to go homepage
-  await page.goto("/");
-  // It should redirect you to the event-types page
-  await page.waitForSelector("[data-testid=event-types]");
+  test("change password", async ({ page }) => {
+    // Try to go homepage
+    await page.goto("/");
+    // It should redirect you to the event-types page
+    await page.waitForSelector("[data-testid=event-types]");
 
-  // Go to http://localhost:3000/settings/security
-  await page.goto("/settings/security");
+    // Go to http://localhost:3000/settings/security
+    await page.goto("/settings/security");
 
-  // Fill form
-  await page.fill('[name="current_password"]', "pro");
-  await page.fill('[name="new_password"]', "pro1");
-  await page.press('[name="new_password"]', "Enter");
+    // Fill form
+    await page.fill('[name="current_password"]', "pro");
+    await page.fill('[name="new_password"]', "pro1");
+    await page.press('[name="new_password"]', "Enter");
 
-  await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible();
+    await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible();
 
-  // Let's revert back to prevent errors on other tests
-  await page.fill('[name="current_password"]', "pro1");
-  await page.fill('[name="new_password"]', "pro");
-  await page.press('[name="new_password"]', "Enter");
+    // Let's revert back to prevent errors on other tests
+    await page.fill('[name="current_password"]', "pro1");
+    await page.fill('[name="new_password"]', "pro");
+    await page.press('[name="new_password"]', "Enter");
 
-  await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible();
+    await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible();
+  });
 });

apps/web/playwright/event-types.test.ts

@@ -1,81 +1,92 @@
 import { expect, test } from "@playwright/test";
 
 import { randomString } from "../lib/random";
+import { deleteEventTypeByTitle } from "./lib/teardown";
 
-test.beforeEach(async ({ page }) => {
-  await page.goto("/event-types");
-  // We wait until loading is finished
-  await page.waitForSelector('[data-testid="event-types"]');
-});
-
-test.describe("pro user", () => {
-  test.use({ storageState: "playwright/artifacts/proStorageState.json" });
-
-  test("has at least 2 events", async ({ page }) => {
-    const $eventTypes = await page.$$("[data-testid=event-types] > *");
-
-    expect($eventTypes.length).toBeGreaterThanOrEqual(2);
-    for (const $el of $eventTypes) {
-      expect(await $el.getAttribute("data-disabled")).toBe("0");
-    }
+test.describe("Event Types tests", () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto("/event-types");
+    // We wait until loading is finished
+    await page.waitForSelector('[data-testid="event-types"]');
   });
 
-  test("can add new event type", async ({ page }) => {
-    await page.click("[data-testid=new-event-type]");
-    const nonce = randomString(3);
-    const eventTitle = `hello ${nonce}`;
+  test.describe("pro user", () => {
+    let isCreated;
+    let eventTitle;
 
-    await page.fill("[name=title]", eventTitle);
-    await page.fill("[name=length]", "10");
-    await page.click("[type=submit]");
+    test.afterAll(async () => {
+      if (isCreated) await deleteEventTypeByTitle(eventTitle);
+    });
+    test.use({ storageState: "playwright/artifacts/proStorageState.json" });
 
-    await page.waitForNavigation({
-      url(url) {
-        return url.pathname !== "/event-types";
-      },
+    test("has at least 2 events", async ({ page }) => {
+      const $eventTypes = await page.locator("[data-testid=event-types] > *");
+      const count = await $eventTypes.count();
+      expect(count).toBeGreaterThanOrEqual(2);
+
+      for (let i = 0; i < count; i++) {
+        expect(await $eventTypes.nth(i).getAttribute("data-disabled")).toBe("0");
+      }
     });
 
-    await page.goto("/event-types");
+    test("can add new event type", async ({ page }) => {
+      await page.click("[data-testid=new-event-type]");
+      const nonce = randomString(3);
+      eventTitle = `hello ${nonce}`;
 
-    await expect(page.locator(`text='${eventTitle}'`)).toBeVisible();
+      await page.fill("[name=title]", eventTitle);
+      await page.fill("[name=length]", "10");
+      await page.click("[type=submit]");
+
+      await page.waitForNavigation({
+        url(url) {
+          return url.pathname !== "/event-types";
+        },
+      });
+
+      await page.goto("/event-types");
+
+      isCreated = await expect(page.locator(`text='${eventTitle}'`)).toBeVisible();
+    });
+
+    test("can duplicate an existing event type", async ({ page }) => {
+      const firstTitle = await page.locator("[data-testid=event-type-title-3]").innerText();
+      const firstFullSlug = await page.locator("[data-testid=event-type-slug-3]").innerText();
+      const firstSlug = firstFullSlug.split("/")[2];
+
+      await page.click("[data-testid=event-type-options-3]");
+      await page.click("[data-testid=event-type-duplicate-3]");
+
+      const url = await page.url();
+      const params = new URLSearchParams(url);
+
+      await expect(params.get("title")).toBe(firstTitle);
+      await expect(params.get("slug")).toBe(firstSlug);
+
+      const formTitle = await page.inputValue("[name=title]");
+      const formSlug = await page.inputValue("[name=slug]");
+
+      await expect(formTitle).toBe(firstTitle);
+      await expect(formSlug).toBe(firstSlug);
+    });
   });
 
-  test("can duplicate an existing event type", async ({ page }) => {
-    const firstTitle = await page.locator("[data-testid=event-type-title-3]").innerText();
-    const firstFullSlug = await page.locator("[data-testid=event-type-slug-3]").innerText();
-    const firstSlug = firstFullSlug.split("/")[2];
+  test.describe("free user", () => {
+    test.use({ storageState: "playwright/artifacts/freeStorageState.json" });
 
-    await page.click("[data-testid=event-type-options-3]");
-    await page.click("[data-testid=event-type-duplicate-3]");
+    test("has at least 2 events where first is enabled", async ({ page }) => {
+      const $eventTypes = await page.locator("[data-testid=event-types] > *");
+      const count = await $eventTypes.count();
+      expect(count).toBeGreaterThanOrEqual(2);
 
-    const url = await page.url();
-    const params = new URLSearchParams(url);
+      const $first = await $eventTypes.first();
+      const $last = await $eventTypes.last()!;
+      expect(await $first.getAttribute("data-disabled")).toBe("0");
+      expect(await $last.getAttribute("data-disabled")).toBe("1");
+    });
 
-    await expect(params.get("title")).toBe(firstTitle);
-    await expect(params.get("slug")).toBe(firstSlug);
-
-    const formTitle = await page.inputValue("[name=title]");
-    const formSlug = await page.inputValue("[name=slug]");
-
-    await expect(formTitle).toBe(firstTitle);
-    await expect(formSlug).toBe(firstSlug);
-  });
-});
-
-test.describe("free user", () => {
-  test.use({ storageState: "playwright/artifacts/freeStorageState.json" });
-
-  test("has at least 2 events where first is enabled", async ({ page }) => {
-    const $eventTypes = await page.$$("[data-testid=event-types] > *");
-
-    expect($eventTypes.length).toBeGreaterThanOrEqual(2);
-    const [$first] = $eventTypes;
-    const $last = $eventTypes.pop()!;
-    expect(await $first.getAttribute("data-disabled")).toBe("0");
-    expect(await $last.getAttribute("data-disabled")).toBe("1");
-  });
-
-  test("can not add new event type", async ({ page }) => {
-    await expect(page.locator("[data-testid=new-event-type]")).toBeDisabled();
+    test("can not add new event type", async ({ page }) => {
+      await expect(page.locator("[data-testid=new-event-type]")).toBeDisabled();
+    });
   });
 });

apps/web/playwright/integrations-stripe.test.ts

@@ -1,9 +1,14 @@
 import { expect, test } from "@playwright/test";
 
 import { hasIntegrationInstalled } from "../lib/integrations/getIntegrations";
+import * as teardown from "./lib/teardown";
 import { selectFirstAvailableTimeSlotNextMonth, todo } from "./lib/testUtils";
 
 test.describe.serial("Stripe integration", () => {
+  test.afterAll(() => {
+    teardown.deleteAllPaymentsByEmail("pro@example.com");
+    teardown.deleteAllBookingsByEmail("pro@example.com");
+  });
   test.skip(!hasIntegrationInstalled("stripe_payment"), "It should only run if Stripe is installed");
 
   test.describe.serial("Stripe integration dashboard", () => {

apps/web/playwright/integrations.test.ts

@@ -1,8 +1,14 @@
 import { expect, test } from "@playwright/test";
 
+import * as teardown from "./lib/teardown";
 import { createHttpServer, selectFirstAvailableTimeSlotNextMonth, todo, waitFor } from "./lib/testUtils";
 
 test.describe("integrations", () => {
+  //teardown
+  test.afterAll(async () => {
+    await teardown.deleteAllWebhooksByEmail("pro@example.com");
+    await teardown.deleteAllBookingsByEmail("pro@example.com");
+  });
   test.use({ storageState: "playwright/artifacts/proStorageState.json" });
 
   test.beforeEach(async ({ page }) => {

apps/web/playwright/lib/teardown.ts

@@ -0,0 +1,40 @@
+import prisma from "@lib/prisma";
+
+export const deleteAllBookingsByEmail = async (email: string) =>
+  prisma.booking.deleteMany({
+    where: {
+      user: {
+        email,
+      },
+    },
+  });
+
+export const deleteEventTypeByTitle = async (title: string) => {
+  const event = (await prisma.eventType.findFirst({
+    select: { id: true },
+    where: { title: title },
+  }))!;
+  await prisma.eventType.delete({ where: { id: event.id } });
+};
+
+export const deleteAllWebhooksByEmail = async (email: string) => {
+  await prisma.webhook.deleteMany({
+    where: {
+      user: {
+        email,
+      },
+    },
+  });
+};
+
+export const deleteAllPaymentsByEmail = async (email: string) => {
+  await prisma.payment.deleteMany({
+    where: {
+      booking: {
+        user: {
+          email,
+        },
+      },
+    },
+  });
+};

apps/web/playwright/login.test.ts

@@ -1,11 +1,13 @@
 import { test } from "@playwright/test";
 
-// Using logged in state from globalSteup
-test.use({ storageState: "playwright/artifacts/proStorageState.json" });
+test.describe("Login tests", () => {
+  // Using logged in state from globalSteup
+  test.use({ storageState: "playwright/artifacts/proStorageState.json" });
 
-test("login with pro@example.com", async ({ page }) => {
-  // Try to go homepage
-  await page.goto("/");
-  // It should redirect you to the event-types page
-  await page.waitForSelector("[data-testid=event-types]");
+  test("login with pro@example.com", async ({ page }) => {
+    // Try to go homepage
+    await page.goto("/");
+    // It should redirect you to the event-types page
+    await page.waitForSelector("[data-testid=event-types]");
+  });
 });

apps/web/playwright/saml.test.ts

@@ -2,13 +2,15 @@ import { test } from "@playwright/test";
 
 import { IS_SAML_LOGIN_ENABLED } from "../server/lib/constants";
 
-// Using logged in state from globalSteup
-test.use({ storageState: "playwright/artifacts/proStorageState.json" });
+test.describe("SAML tests", () => {
+  // Using logged in state from globalSteup
+  test.use({ storageState: "playwright/artifacts/proStorageState.json" });
 
-test("test SAML configuration UI with pro@example.com", async ({ page }) => {
-  test.skip(!IS_SAML_LOGIN_ENABLED, "It should only run if SAML is enabled");
-  // Try to go Security page
-  await page.goto("/settings/security");
-  // It should redirect you to the event-types page
-  await page.waitForSelector("[data-testid=saml_config]");
+  test("test SAML configuration UI with pro@example.com", async ({ page }) => {
+    test.skip(!IS_SAML_LOGIN_ENABLED, "It should only run if SAML is enabled");
+    // Try to go Security page
+    await page.goto("/settings/security");
+    // It should redirect you to the event-types page
+    await page.waitForSelector("[data-testid=saml_config]");
+  });
 });

apps/web/playwright/trial.test.ts

@@ -1,13 +1,15 @@
 import { expect, test } from "@playwright/test";
 
-// Using logged in state from globalSteup
-test.use({ storageState: "playwright/artifacts/trialStorageState.json" });
+test.describe("Trial account tests", () => {
+  // Using logged in state from globalSteup
+  test.use({ storageState: "playwright/artifacts/trialStorageState.json" });
 
-test("Trial banner should be visible to TRIAL users", async ({ page }) => {
-  // Try to go homepage
-  await page.goto("/");
-  // It should redirect you to the event-types page
-  await page.waitForSelector("[data-testid=event-types]");
+  test("Trial banner should be visible to TRIAL users", async ({ page }) => {
+    // Try to go homepage
+    await page.goto("/");
+    // It should redirect you to the event-types page
+    await page.waitForSelector("[data-testid=event-types]");
 
-  await expect(page.locator(`[data-testid=trial-banner]`)).toBeVisible();
+    await expect(page.locator(`[data-testid=trial-banner]`)).toBeVisible();
+  });
 });

package.json

@@ -26,7 +26,7 @@
     "prepare": "husky install",
     "start": "turbo run start --scope=\"@calcom/web\"",
     "test": "turbo run test",
-    "test-playwright": "yarn playwright test",
+    "test-playwright": "yarn playwright test --config=tests/config/playwright.config.ts",
     "test-e2e": "turbo run test-e2e --concurrency=1",
     "type-check": "turbo run type-check"
   },

tests/config/playwright.config.ts

@@ -19,6 +19,8 @@ const testDir = path.join(__dirname, "..", "..", "apps/web/playwright");
 
 const config: PlaywrightTestConfig = {
   forbidOnly: !!process.env.CI,
+  retries: 1,
+  workers: 1,
   timeout: 60_000,
   reporter: [
     [process.env.CI ? "github" : "list"],
@@ -28,7 +30,7 @@ const config: PlaywrightTestConfig = {
   globalSetup: require.resolve("./globalSetup"),
   outputDir,
   webServer: {
-    command: "yarn workspace @calcom/web start -p 3000",
+    command: "NEXT_PUBLIC_IS_E2E=1 yarn workspace @calcom/web start -p 3000",
     port: 3000,
     timeout: 60_000,
     reuseExistingServer: !process.env.CI,

turbo.json

@@ -20,7 +20,6 @@
     "@calcom/web#build": {
       "dependsOn": [
         "^build",
-        "@calcom/prisma#db-deploy",
         "$BASE_URL",
         "$CALENDSO_ENCRYPTION_KEY",
         "$CRON_API_KEY",
@@ -46,7 +45,6 @@
         "$PAYMENT_FEE_FIXED",
         "$PAYMENT_FEE_PERCENTAGE",
         "$PGSSLMODE",
-        "$PLAYWRIGHT_SECRET",
         "$SAML_ADMINS",
         "$SAML_DATABASE_URL",
         "$STRIPE_CLIENT_ID",
@@ -64,7 +62,9 @@
     "@calcom/web#dx": {
       "dependsOn": ["@calcom/prisma#dx"]
     },
-    "@calcom/web#start": {},
+    "@calcom/web#start": {
+      "dependsOn": ["@calcom/prisma#db-deploy"]
+    },
     "@calcom/website#build": {
       "dependsOn": ["$WEBSITE_BASE_URL"],
       "outputs": [".next/**"]
@@ -97,12 +97,11 @@
     },
     "start": {},
     "test": {
-      "dependsOn": []
+      "dependsOn": ["^test"]
     },
     "test-e2e": {
       "cache": false,
-      "dependsOn": ["^test", "@calcom/web#build", "@calcom/prisma#db-reset"],
-      "outputs": ["playwright", "test-results"]
+      "dependsOn": ["test", "@calcom/web#build", "@calcom/prisma#db-reset"]
     },
     "type-check": {
       "outputs": []