From 55587e92c14fb8c82adb3ea4b73e4831219da5bf Mon Sep 17 00:00:00 2001 From: Demian Caldelas Date: Thu, 17 Mar 2022 16:36:11 -0300 Subject: [PATCH] Fix a set of E2E bugs causing several CI failures (#2177) * Fix E2E bugs causing CI failutes * Revert setup in dx Co-authored-by: zomars --- .github/workflows/e2e.yml | 1 - README.md | 2 +- apps/web/.env.example | 6 +- apps/web/pages/api/auth/forgot-password.ts | 5 +- .../playwright/auth/forgot-password.test.ts | 15 +- apps/web/playwright/booking-pages.test.ts | 23 +-- apps/web/playwright/change-password.test.ts | 40 ++--- apps/web/playwright/event-types.test.ts | 137 ++++++++++-------- .../playwright/integrations-stripe.test.ts | 5 + apps/web/playwright/integrations.test.ts | 6 + apps/web/playwright/lib/teardown.ts | 40 +++++ apps/web/playwright/login.test.ts | 16 +- apps/web/playwright/saml.test.ts | 18 ++- apps/web/playwright/trial.test.ts | 18 ++- package.json | 2 +- packages/prisma/zod/custom/eventtype.ts | 0 packages/prisma/zod/webhook.ts | 0 tests/config/playwright.config.ts | 4 +- turbo.json | 11 +- 19 files changed, 198 insertions(+), 151 deletions(-) create mode 100644 apps/web/playwright/lib/teardown.ts mode change 100644 => 100755 packages/prisma/zod/custom/eventtype.ts mode change 100644 => 100755 packages/prisma/zod/webhook.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4050ff588f..0430294af9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -13,7 +13,6 @@ jobs: DATABASE_URL: postgresql://postgres:@localhost:5432/calendso BASE_URL: http://localhost:3000 JWT_SECRET: secret - PLAYWRIGHT_SECRET: ${{ secrets.CI_PLAYWRIGHT_SECRET }} GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }} GOOGLE_LOGIN_ENABLED: true # CRON_API_KEY: xxx diff --git a/README.md b/README.md index 2fb0e35716..a0b681dfc1 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ echo 'NEXT_PUBLIC_DEBUG=1' >> apps/web/.env ### E2E-Testing ```sh -# In first terminal +# In first terminal. Must run on port 3000. yarn dx # In second terminal yarn workspace @calcom/web test-e2e diff --git a/apps/web/.env.example b/apps/web/.env.example index db2e770a3a..424cd658b4 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -25,7 +25,6 @@ NEXT_PUBLIC_APP_URL='http://localhost:3000' JWT_SECRET='secret' # This is used so we can bypass emails in auth flows for E2E testing -PLAYWRIGHT_SECRET= # To enable SAML login, set both these variables # @see https://github.com/calcom/cal.com/tree/main/packages/ee#setting-up-saml-login @@ -101,4 +100,7 @@ CALENDSO_ENCRYPTION_KEY= NEXT_PUBLIC_INTERCOM_APP_ID= # Zendesk Config -NEXT_PUBLIC_ZENDESK_KEY= \ No newline at end of file +NEXT_PUBLIC_ZENDESK_KEY= + +# Set it to "1" if you need to run E2E tests locally +NEXT_PUBLIC_IS_E2E= diff --git a/apps/web/pages/api/auth/forgot-password.ts b/apps/web/pages/api/auth/forgot-password.ts index c0ad8ca6ee..e415e528de 100644 --- a/apps/web/pages/api/auth/forgot-password.ts +++ b/apps/web/pages/api/auth/forgot-password.ts @@ -65,10 +65,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await sendPasswordResetEmail(passwordEmail); /** So we can test the password reset flow on CI */ - if ( - process.env.PLAYWRIGHT_SECRET && - req.headers["x-playwright-secret"] === process.env.PLAYWRIGHT_SECRET - ) { + if (process.env.NEXT_PUBLIC_IS_E2E) { return res.status(201).json({ message: "Reset Requested", resetLink }); } diff --git a/apps/web/playwright/auth/forgot-password.test.ts b/apps/web/playwright/auth/forgot-password.test.ts index 9d336130ff..87921a61d6 100644 --- a/apps/web/playwright/auth/forgot-password.test.ts +++ b/apps/web/playwright/auth/forgot-password.test.ts @@ -1,17 +1,6 @@ import { expect, test } from "@playwright/test"; -test("Can reset forgotten password", async ({ browser }) => { - test.fixme(true, "TODO: This test is failing randomly, disabled for now"); - // Create a new incognito browser context - const context = await browser.newContext({ - extraHTTPHeaders: { - // Only needed for bypassing emails while testing - "X-Playwright-Secret": process.env.PLAYWRIGHT_SECRET || "", - }, - }); - // Create a new page inside context. - const page = await context.newPage(); - +test("Can reset forgotten password", async ({ page }) => { // Got to reset password flow await page.goto("/auth/forgot-password"); @@ -50,6 +39,4 @@ test("Can reset forgotten password", async ({ browser }) => { await page.waitForSelector("[data-testid=dashboard-shell]"); await expect(page.locator("[data-testid=dashboard-shell]")).toBeVisible(); - - await context.close(); }); diff --git a/apps/web/playwright/booking-pages.test.ts b/apps/web/playwright/booking-pages.test.ts index 6c19762994..cbed0460c8 100644 --- a/apps/web/playwright/booking-pages.test.ts +++ b/apps/web/playwright/booking-pages.test.ts @@ -1,18 +1,8 @@ import { expect, test } from "@playwright/test"; -import prisma from "@lib/prisma"; - +import { deleteAllBookingsByEmail } from "./lib/teardown"; import { selectFirstAvailableTimeSlotNextMonth, todo } from "./lib/testUtils"; -const deleteBookingsByEmail = async (email: string) => - prisma.booking.deleteMany({ - where: { - user: { - email, - }, - }, - }); - test.describe("free user", () => { test.beforeEach(async ({ page }) => { await page.goto("/free"); @@ -20,7 +10,7 @@ test.describe("free user", () => { test.afterEach(async () => { // delete test bookings - await deleteBookingsByEmail("free@example.com"); + await deleteAllBookingsByEmail("free@example.com"); }); test("only one visible event", async ({ page }) => { @@ -81,14 +71,15 @@ test.describe("pro user", () => { await page.goto("/pro"); }); - test.afterEach(async () => { + test.afterAll(async () => { // delete test bookings - await deleteBookingsByEmail("pro@example.com"); + await deleteAllBookingsByEmail("pro@example.com"); }); test("pro user's page has at least 2 visible events", async ({ page }) => { - const $eventTypes = await page.$$("[data-testid=event-types] > *"); - expect($eventTypes.length).toBeGreaterThanOrEqual(2); + // await page.pause(); + const $eventTypes = await page.locator("[data-testid=event-types] > *"); + expect(await $eventTypes.count()).toBeGreaterThanOrEqual(2); }); test("book an event first day in next month", async ({ page }) => { diff --git a/apps/web/playwright/change-password.test.ts b/apps/web/playwright/change-password.test.ts index 11ea037a07..f69ff704c1 100644 --- a/apps/web/playwright/change-password.test.ts +++ b/apps/web/playwright/change-password.test.ts @@ -1,28 +1,30 @@ import { expect, test } from "@playwright/test"; -// Using logged in state from globalSteup -test.use({ storageState: "playwright/artifacts/proStorageState.json" }); +test.describe("Change Passsword Test", () => { + // Using logged in state from globalSteup + test.use({ storageState: "playwright/artifacts/proStorageState.json" }); -test("change password", async ({ page }) => { - // Try to go homepage - await page.goto("/"); - // It should redirect you to the event-types page - await page.waitForSelector("[data-testid=event-types]"); + test("change password", async ({ page }) => { + // Try to go homepage + await page.goto("/"); + // It should redirect you to the event-types page + await page.waitForSelector("[data-testid=event-types]"); - // Go to http://localhost:3000/settings/security - await page.goto("/settings/security"); + // Go to http://localhost:3000/settings/security + await page.goto("/settings/security"); - // Fill form - await page.fill('[name="current_password"]', "pro"); - await page.fill('[name="new_password"]', "pro1"); - await page.press('[name="new_password"]', "Enter"); + // Fill form + await page.fill('[name="current_password"]', "pro"); + await page.fill('[name="new_password"]', "pro1"); + await page.press('[name="new_password"]', "Enter"); - await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible(); + await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible(); - // Let's revert back to prevent errors on other tests - await page.fill('[name="current_password"]', "pro1"); - await page.fill('[name="new_password"]', "pro"); - await page.press('[name="new_password"]', "Enter"); + // Let's revert back to prevent errors on other tests + await page.fill('[name="current_password"]', "pro1"); + await page.fill('[name="new_password"]', "pro"); + await page.press('[name="new_password"]', "Enter"); - await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible(); + await expect(page.locator(`text=Your password has been successfully changed.`)).toBeVisible(); + }); }); diff --git a/apps/web/playwright/event-types.test.ts b/apps/web/playwright/event-types.test.ts index 9f68d4ac75..dd67ad4b01 100644 --- a/apps/web/playwright/event-types.test.ts +++ b/apps/web/playwright/event-types.test.ts @@ -1,81 +1,92 @@ import { expect, test } from "@playwright/test"; import { randomString } from "../lib/random"; +import { deleteEventTypeByTitle } from "./lib/teardown"; -test.beforeEach(async ({ page }) => { - await page.goto("/event-types"); - // We wait until loading is finished - await page.waitForSelector('[data-testid="event-types"]'); -}); - -test.describe("pro user", () => { - test.use({ storageState: "playwright/artifacts/proStorageState.json" }); - - test("has at least 2 events", async ({ page }) => { - const $eventTypes = await page.$$("[data-testid=event-types] > *"); - - expect($eventTypes.length).toBeGreaterThanOrEqual(2); - for (const $el of $eventTypes) { - expect(await $el.getAttribute("data-disabled")).toBe("0"); - } +test.describe("Event Types tests", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/event-types"); + // We wait until loading is finished + await page.waitForSelector('[data-testid="event-types"]'); }); - test("can add new event type", async ({ page }) => { - await page.click("[data-testid=new-event-type]"); - const nonce = randomString(3); - const eventTitle = `hello ${nonce}`; + test.describe("pro user", () => { + let isCreated; + let eventTitle; - await page.fill("[name=title]", eventTitle); - await page.fill("[name=length]", "10"); - await page.click("[type=submit]"); + test.afterAll(async () => { + if (isCreated) await deleteEventTypeByTitle(eventTitle); + }); + test.use({ storageState: "playwright/artifacts/proStorageState.json" }); - await page.waitForNavigation({ - url(url) { - return url.pathname !== "/event-types"; - }, + test("has at least 2 events", async ({ page }) => { + const $eventTypes = await page.locator("[data-testid=event-types] > *"); + const count = await $eventTypes.count(); + expect(count).toBeGreaterThanOrEqual(2); + + for (let i = 0; i < count; i++) { + expect(await $eventTypes.nth(i).getAttribute("data-disabled")).toBe("0"); + } }); - await page.goto("/event-types"); + test("can add new event type", async ({ page }) => { + await page.click("[data-testid=new-event-type]"); + const nonce = randomString(3); + eventTitle = `hello ${nonce}`; - await expect(page.locator(`text='${eventTitle}'`)).toBeVisible(); + await page.fill("[name=title]", eventTitle); + await page.fill("[name=length]", "10"); + await page.click("[type=submit]"); + + await page.waitForNavigation({ + url(url) { + return url.pathname !== "/event-types"; + }, + }); + + await page.goto("/event-types"); + + isCreated = await expect(page.locator(`text='${eventTitle}'`)).toBeVisible(); + }); + + test("can duplicate an existing event type", async ({ page }) => { + const firstTitle = await page.locator("[data-testid=event-type-title-3]").innerText(); + const firstFullSlug = await page.locator("[data-testid=event-type-slug-3]").innerText(); + const firstSlug = firstFullSlug.split("/")[2]; + + await page.click("[data-testid=event-type-options-3]"); + await page.click("[data-testid=event-type-duplicate-3]"); + + const url = await page.url(); + const params = new URLSearchParams(url); + + await expect(params.get("title")).toBe(firstTitle); + await expect(params.get("slug")).toBe(firstSlug); + + const formTitle = await page.inputValue("[name=title]"); + const formSlug = await page.inputValue("[name=slug]"); + + await expect(formTitle).toBe(firstTitle); + await expect(formSlug).toBe(firstSlug); + }); }); - test("can duplicate an existing event type", async ({ page }) => { - const firstTitle = await page.locator("[data-testid=event-type-title-3]").innerText(); - const firstFullSlug = await page.locator("[data-testid=event-type-slug-3]").innerText(); - const firstSlug = firstFullSlug.split("/")[2]; + test.describe("free user", () => { + test.use({ storageState: "playwright/artifacts/freeStorageState.json" }); - await page.click("[data-testid=event-type-options-3]"); - await page.click("[data-testid=event-type-duplicate-3]"); + test("has at least 2 events where first is enabled", async ({ page }) => { + const $eventTypes = await page.locator("[data-testid=event-types] > *"); + const count = await $eventTypes.count(); + expect(count).toBeGreaterThanOrEqual(2); - const url = await page.url(); - const params = new URLSearchParams(url); + const $first = await $eventTypes.first(); + const $last = await $eventTypes.last()!; + expect(await $first.getAttribute("data-disabled")).toBe("0"); + expect(await $last.getAttribute("data-disabled")).toBe("1"); + }); - await expect(params.get("title")).toBe(firstTitle); - await expect(params.get("slug")).toBe(firstSlug); - - const formTitle = await page.inputValue("[name=title]"); - const formSlug = await page.inputValue("[name=slug]"); - - await expect(formTitle).toBe(firstTitle); - await expect(formSlug).toBe(firstSlug); - }); -}); - -test.describe("free user", () => { - test.use({ storageState: "playwright/artifacts/freeStorageState.json" }); - - test("has at least 2 events where first is enabled", async ({ page }) => { - const $eventTypes = await page.$$("[data-testid=event-types] > *"); - - expect($eventTypes.length).toBeGreaterThanOrEqual(2); - const [$first] = $eventTypes; - const $last = $eventTypes.pop()!; - expect(await $first.getAttribute("data-disabled")).toBe("0"); - expect(await $last.getAttribute("data-disabled")).toBe("1"); - }); - - test("can not add new event type", async ({ page }) => { - await expect(page.locator("[data-testid=new-event-type]")).toBeDisabled(); + test("can not add new event type", async ({ page }) => { + await expect(page.locator("[data-testid=new-event-type]")).toBeDisabled(); + }); }); }); diff --git a/apps/web/playwright/integrations-stripe.test.ts b/apps/web/playwright/integrations-stripe.test.ts index 225f171ed1..14a7e51479 100644 --- a/apps/web/playwright/integrations-stripe.test.ts +++ b/apps/web/playwright/integrations-stripe.test.ts @@ -1,9 +1,14 @@ import { expect, test } from "@playwright/test"; import { hasIntegrationInstalled } from "../lib/integrations/getIntegrations"; +import * as teardown from "./lib/teardown"; import { selectFirstAvailableTimeSlotNextMonth, todo } from "./lib/testUtils"; test.describe.serial("Stripe integration", () => { + test.afterAll(() => { + teardown.deleteAllPaymentsByEmail("pro@example.com"); + teardown.deleteAllBookingsByEmail("pro@example.com"); + }); test.skip(!hasIntegrationInstalled("stripe_payment"), "It should only run if Stripe is installed"); test.describe.serial("Stripe integration dashboard", () => { diff --git a/apps/web/playwright/integrations.test.ts b/apps/web/playwright/integrations.test.ts index 88bd8f7552..1d2089c537 100644 --- a/apps/web/playwright/integrations.test.ts +++ b/apps/web/playwright/integrations.test.ts @@ -1,8 +1,14 @@ import { expect, test } from "@playwright/test"; +import * as teardown from "./lib/teardown"; import { createHttpServer, selectFirstAvailableTimeSlotNextMonth, todo, waitFor } from "./lib/testUtils"; test.describe("integrations", () => { + //teardown + test.afterAll(async () => { + await teardown.deleteAllWebhooksByEmail("pro@example.com"); + await teardown.deleteAllBookingsByEmail("pro@example.com"); + }); test.use({ storageState: "playwright/artifacts/proStorageState.json" }); test.beforeEach(async ({ page }) => { diff --git a/apps/web/playwright/lib/teardown.ts b/apps/web/playwright/lib/teardown.ts new file mode 100644 index 0000000000..51fbbb61bf --- /dev/null +++ b/apps/web/playwright/lib/teardown.ts @@ -0,0 +1,40 @@ +import prisma from "@lib/prisma"; + +export const deleteAllBookingsByEmail = async (email: string) => + prisma.booking.deleteMany({ + where: { + user: { + email, + }, + }, + }); + +export const deleteEventTypeByTitle = async (title: string) => { + const event = (await prisma.eventType.findFirst({ + select: { id: true }, + where: { title: title }, + }))!; + await prisma.eventType.delete({ where: { id: event.id } }); +}; + +export const deleteAllWebhooksByEmail = async (email: string) => { + await prisma.webhook.deleteMany({ + where: { + user: { + email, + }, + }, + }); +}; + +export const deleteAllPaymentsByEmail = async (email: string) => { + await prisma.payment.deleteMany({ + where: { + booking: { + user: { + email, + }, + }, + }, + }); +}; diff --git a/apps/web/playwright/login.test.ts b/apps/web/playwright/login.test.ts index dc77e6c581..f89f2f64aa 100644 --- a/apps/web/playwright/login.test.ts +++ b/apps/web/playwright/login.test.ts @@ -1,11 +1,13 @@ import { test } from "@playwright/test"; -// Using logged in state from globalSteup -test.use({ storageState: "playwright/artifacts/proStorageState.json" }); +test.describe("Login tests", () => { + // Using logged in state from globalSteup + test.use({ storageState: "playwright/artifacts/proStorageState.json" }); -test("login with pro@example.com", async ({ page }) => { - // Try to go homepage - await page.goto("/"); - // It should redirect you to the event-types page - await page.waitForSelector("[data-testid=event-types]"); + test("login with pro@example.com", async ({ page }) => { + // Try to go homepage + await page.goto("/"); + // It should redirect you to the event-types page + await page.waitForSelector("[data-testid=event-types]"); + }); }); diff --git a/apps/web/playwright/saml.test.ts b/apps/web/playwright/saml.test.ts index fd34ff3347..56120bfc90 100644 --- a/apps/web/playwright/saml.test.ts +++ b/apps/web/playwright/saml.test.ts @@ -2,13 +2,15 @@ import { test } from "@playwright/test"; import { IS_SAML_LOGIN_ENABLED } from "../server/lib/constants"; -// Using logged in state from globalSteup -test.use({ storageState: "playwright/artifacts/proStorageState.json" }); +test.describe("SAML tests", () => { + // Using logged in state from globalSteup + test.use({ storageState: "playwright/artifacts/proStorageState.json" }); -test("test SAML configuration UI with pro@example.com", async ({ page }) => { - test.skip(!IS_SAML_LOGIN_ENABLED, "It should only run if SAML is enabled"); - // Try to go Security page - await page.goto("/settings/security"); - // It should redirect you to the event-types page - await page.waitForSelector("[data-testid=saml_config]"); + test("test SAML configuration UI with pro@example.com", async ({ page }) => { + test.skip(!IS_SAML_LOGIN_ENABLED, "It should only run if SAML is enabled"); + // Try to go Security page + await page.goto("/settings/security"); + // It should redirect you to the event-types page + await page.waitForSelector("[data-testid=saml_config]"); + }); }); diff --git a/apps/web/playwright/trial.test.ts b/apps/web/playwright/trial.test.ts index debb80eae5..c509939183 100644 --- a/apps/web/playwright/trial.test.ts +++ b/apps/web/playwright/trial.test.ts @@ -1,13 +1,15 @@ import { expect, test } from "@playwright/test"; -// Using logged in state from globalSteup -test.use({ storageState: "playwright/artifacts/trialStorageState.json" }); +test.describe("Trial account tests", () => { + // Using logged in state from globalSteup + test.use({ storageState: "playwright/artifacts/trialStorageState.json" }); -test("Trial banner should be visible to TRIAL users", async ({ page }) => { - // Try to go homepage - await page.goto("/"); - // It should redirect you to the event-types page - await page.waitForSelector("[data-testid=event-types]"); + test("Trial banner should be visible to TRIAL users", async ({ page }) => { + // Try to go homepage + await page.goto("/"); + // It should redirect you to the event-types page + await page.waitForSelector("[data-testid=event-types]"); - await expect(page.locator(`[data-testid=trial-banner]`)).toBeVisible(); + await expect(page.locator(`[data-testid=trial-banner]`)).toBeVisible(); + }); }); diff --git a/package.json b/package.json index 451147de01..9cb92f6047 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "prepare": "husky install", "start": "turbo run start --scope=\"@calcom/web\"", "test": "turbo run test", - "test-playwright": "yarn playwright test", + "test-playwright": "yarn playwright test --config=tests/config/playwright.config.ts", "test-e2e": "turbo run test-e2e --concurrency=1", "type-check": "turbo run type-check" }, diff --git a/packages/prisma/zod/custom/eventtype.ts b/packages/prisma/zod/custom/eventtype.ts old mode 100644 new mode 100755 diff --git a/packages/prisma/zod/webhook.ts b/packages/prisma/zod/webhook.ts old mode 100644 new mode 100755 diff --git a/tests/config/playwright.config.ts b/tests/config/playwright.config.ts index 09b5a136b9..9534c9ac0a 100644 --- a/tests/config/playwright.config.ts +++ b/tests/config/playwright.config.ts @@ -19,6 +19,8 @@ const testDir = path.join(__dirname, "..", "..", "apps/web/playwright"); const config: PlaywrightTestConfig = { forbidOnly: !!process.env.CI, + retries: 1, + workers: 1, timeout: 60_000, reporter: [ [process.env.CI ? "github" : "list"], @@ -28,7 +30,7 @@ const config: PlaywrightTestConfig = { globalSetup: require.resolve("./globalSetup"), outputDir, webServer: { - command: "yarn workspace @calcom/web start -p 3000", + command: "NEXT_PUBLIC_IS_E2E=1 yarn workspace @calcom/web start -p 3000", port: 3000, timeout: 60_000, reuseExistingServer: !process.env.CI, diff --git a/turbo.json b/turbo.json index 5590758dce..b957d9b606 100644 --- a/turbo.json +++ b/turbo.json @@ -20,7 +20,6 @@ "@calcom/web#build": { "dependsOn": [ "^build", - "@calcom/prisma#db-deploy", "$BASE_URL", "$CALENDSO_ENCRYPTION_KEY", "$CRON_API_KEY", @@ -46,7 +45,6 @@ "$PAYMENT_FEE_FIXED", "$PAYMENT_FEE_PERCENTAGE", "$PGSSLMODE", - "$PLAYWRIGHT_SECRET", "$SAML_ADMINS", "$SAML_DATABASE_URL", "$STRIPE_CLIENT_ID", @@ -64,7 +62,9 @@ "@calcom/web#dx": { "dependsOn": ["@calcom/prisma#dx"] }, - "@calcom/web#start": {}, + "@calcom/web#start": { + "dependsOn": ["@calcom/prisma#db-deploy"] + }, "@calcom/website#build": { "dependsOn": ["$WEBSITE_BASE_URL"], "outputs": [".next/**"] @@ -97,12 +97,11 @@ }, "start": {}, "test": { - "dependsOn": [] + "dependsOn": ["^test"] }, "test-e2e": { "cache": false, - "dependsOn": ["^test", "@calcom/web#build", "@calcom/prisma#db-reset"], - "outputs": ["playwright", "test-results"] + "dependsOn": ["test", "@calcom/web#build", "@calcom/prisma#db-reset"] }, "type-check": { "outputs": []