feat: Calendar Cache (#11185)

This commit is contained in:
Omar López 2023-09-14 15:19:39 -07:00 committed by GitHub
parent 9bc40a3eb6
commit c95917cceb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 468 additions and 266 deletions

View File

@ -0,0 +1,16 @@
import type { NextApiRequest, NextApiResponse } from "next";
import prisma from "@calcom/prisma";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
const deleted = await prisma.calendarCache.deleteMany({
where: {
// Delete all cache entries that expired before now
expiresAt: {
lte: new Date(Date.now()),
},
},
});
res.json({ ok: true, count: deleted.count });
}

8
apps/web/vercel.json Normal file
View File

@ -0,0 +1,8 @@
{
"crons": [
{
"path": "/api/cron/calendar-cache-cleanup",
"schedule": "0 5 * * *"
}
]
}

View File

@ -0,0 +1,101 @@
import prismaMock from "../../../../tests/libs/__mocks__/prisma";
import { afterEach, expect, test, vi } from "vitest";
import CalendarService from "./CalendarService";
afterEach(() => {
vi.resetAllMocks();
});
vi.mock("@calcom/features/flags/server/utils", () => ({
getFeatureFlagMap: vi.fn().mockResolvedValue({
"calendar-cache": true,
}),
}));
vi.mock("./getGoogleAppKeys", () => ({
getGoogleAppKeys: vi.fn().mockResolvedValue({
client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com",
client_secret: "xxxxxxxxxxxxxxxxxx",
redirect_uris: ["http://localhost:3000/api/integrations/googlecalendar/callback"],
}),
}));
const googleTestCredential = {
scope: "https://www.googleapis.com/auth/calendar.events",
token_type: "Bearer",
expiry_date: 1625097600000,
access_token: "",
refresh_token: "",
};
const testCredential = {
appId: "test",
id: 1,
invalid: false,
key: googleTestCredential,
type: "test",
userId: 1,
user: { email: "example@cal.com" },
teamId: 1,
};
const testSelectedCalendar = {
userId: 1,
integration: "google_calendar",
externalId: "example@cal.com",
};
const testFreeBusyResponse = {
kind: "calendar#freeBusy",
timeMax: "2024-01-01T20:59:59.000Z",
timeMin: "2023-11-30T20:00:00.000Z",
calendars: {
"example@cal.com": {
busy: [
{ end: "2023-12-01T19:00:00Z", start: "2023-12-01T18:00:00Z" },
{ end: "2023-12-04T19:00:00Z", start: "2023-12-04T18:00:00Z" },
],
},
"xxxxxxxxxxxxxxxxxxxxxxxxxx@group.calendar.google.com": { busy: [] },
},
};
const calendarCacheResponse = {
key: "dummy",
expiresAt: new Date(),
credentialId: 1,
value: testFreeBusyResponse,
};
test("Calendar Cache is being called", async () => {
prismaMock.calendarCache.findUnique
// First call won't have a cache
.mockResolvedValueOnce(null)
// Second call will have a cache
.mockResolvedValueOnce(calendarCacheResponse);
// prismaMock.calendarCache.create.mock.
const calendarService = new CalendarService(testCredential);
// @ts-expect-error authedCalendar is a private method, hence the TS error
vi.spyOn(calendarService, "authedCalendar").mockReturnValue(
// @ts-expect-error trust me bro
{
freebusy: {
query: vi.fn().mockReturnValue({
data: testFreeBusyResponse,
}),
},
}
);
await calendarService.getAvailability(new Date().toISOString(), new Date().toISOString(), [
testSelectedCalendar,
]);
await calendarService.getAvailability(new Date().toISOString(), new Date().toISOString(), [
testSelectedCalendar,
]);
expect(prismaMock.calendarCache.findUnique).toHaveBeenCalled();
expect(prismaMock.calendarCache.upsert).toHaveBeenCalledOnce();
});

View File

@ -4,6 +4,7 @@ import type { calendar_v3 } from "googleapis";
import { google } from "googleapis";
import { MeetLocationType } from "@calcom/app-store/locations";
import { getFeatureFlagMap } from "@calcom/features/flags/server/utils";
import { getLocation, getRichDescription } from "@calcom/lib/CalEventParser";
import type CalendarService from "@calcom/lib/CalendarService";
import logger from "@calcom/lib/logger";
@ -24,6 +25,36 @@ interface GoogleCalError extends Error {
code?: number;
}
const ONE_MINUTE_MS = 60 * 1000;
const CACHING_TIME = ONE_MINUTE_MS;
/** Expand the start date to the start of the month */
function getTimeMin(timeMin: string) {
const dateMin = new Date(timeMin);
return new Date(dateMin.getFullYear(), dateMin.getMonth(), 1, 0, 0, 0, 0).toISOString();
}
/** Expand the end date to the end of the month */
function getTimeMax(timeMax: string) {
const dateMax = new Date(timeMax);
return new Date(dateMax.getFullYear(), dateMax.getMonth() + 1, 0, 0, 0, 0, 0).toISOString();
}
/**
* Enable or disable the expanded cache
* TODO: Make this configurable
* */
const ENABLE_EXPANDED_CACHE = true;
/**
* By expanding the cache to whole months, we can save round trips to the third party APIs.
* In this case we already have the data in the database, so we can just return it.
*/
function handleMinMax(min: string, max: string) {
if (!ENABLE_EXPANDED_CACHE) return { timeMin: min, timeMax: max };
return { timeMin: getTimeMin(min), timeMax: getTimeMax(max) };
}
export default class GoogleCalendarService implements Calendar {
private integrationName = "";
private auth: { getToken: () => Promise<MyGoogleAuth> };
@ -32,6 +63,7 @@ export default class GoogleCalendarService implements Calendar {
constructor(credential: CredentialPayload) {
this.integrationName = "google_calendar";
this.credential = credential;
this.auth = this.googleAuth(credential);
this.log = logger.getChildLogger({ prefix: [`[[lib] ${this.integrationName}`] });
this.credential = credential;
@ -86,6 +118,15 @@ export default class GoogleCalendarService implements Calendar {
};
};
private authedCalendar = async () => {
const myGoogleAuth = await this.auth.getToken();
const calendar = google.calendar({
version: "v3",
auth: myGoogleAuth,
});
return calendar;
};
private getAttendees = (event: CalendarEvent) => {
// When rescheduling events we know the external id of the calendar so we can just look for it in the destinationCalendar array.
const selectedHostDestinationCalendar = event.destinationCalendar?.find(
@ -129,209 +170,236 @@ export default class GoogleCalendarService implements Calendar {
};
async createEvent(calEventRaw: CalendarEvent, credentialId: number): Promise<NewCalendarEventType> {
return new Promise(async (resolve, reject) => {
const myGoogleAuth = await this.auth.getToken();
const payload: calendar_v3.Schema$Event = {
summary: calEventRaw.title,
description: getRichDescription(calEventRaw),
start: {
dateTime: calEventRaw.startTime,
timeZone: calEventRaw.organizer.timeZone,
},
end: {
dateTime: calEventRaw.endTime,
timeZone: calEventRaw.organizer.timeZone,
},
attendees: this.getAttendees(calEventRaw),
reminders: {
useDefault: true,
},
guestsCanSeeOtherGuests: !!calEventRaw.seatsPerTimeSlot ? calEventRaw.seatsShowAttendees : true,
};
const payload: calendar_v3.Schema$Event = {
summary: calEventRaw.title,
description: getRichDescription(calEventRaw),
start: {
dateTime: calEventRaw.startTime,
timeZone: calEventRaw.organizer.timeZone,
},
end: {
dateTime: calEventRaw.endTime,
timeZone: calEventRaw.organizer.timeZone,
},
attendees: this.getAttendees(calEventRaw),
reminders: {
useDefault: true,
},
guestsCanSeeOtherGuests: !!calEventRaw.seatsPerTimeSlot ? calEventRaw.seatsShowAttendees : true,
};
if (calEventRaw.location) {
payload["location"] = getLocation(calEventRaw);
}
if (calEventRaw.location) {
payload["location"] = getLocation(calEventRaw);
}
if (calEventRaw.conferenceData && calEventRaw.location === MeetLocationType) {
payload["conferenceData"] = calEventRaw.conferenceData;
}
const calendar = google.calendar({
version: "v3",
if (calEventRaw.conferenceData && calEventRaw.location === MeetLocationType) {
payload["conferenceData"] = calEventRaw.conferenceData;
}
const calendar = await this.authedCalendar();
// Find in calEventRaw.destinationCalendar the one with the same credentialId
const selectedCalendar =
calEventRaw.destinationCalendar?.find((cal) => cal.credentialId === credentialId)?.externalId ||
"primary";
try {
const event = await calendar.events.insert({
calendarId: selectedCalendar,
requestBody: payload,
conferenceDataVersion: 1,
sendUpdates: "none",
});
// Find in calEventRaw.destinationCalendar the one with the same credentialId
const selectedCalendar =
calEventRaw.destinationCalendar?.find((cal) => cal.credentialId === credentialId)?.externalId ||
"primary";
calendar.events.insert(
{
auth: myGoogleAuth,
if (event && event.data.id && event.data.hangoutLink) {
await calendar.events.patch({
// Update the same event but this time we know the hangout link
calendarId: selectedCalendar,
requestBody: payload,
conferenceDataVersion: 1,
sendUpdates: "none",
},
function (error, event) {
if (error || !event?.data) {
console.error("There was an error contacting google calendar service: ", error);
return reject(error);
}
eventId: event.data.id || "",
requestBody: {
description: getRichDescription({
...calEventRaw,
additionalInformation: { hangoutLink: event.data.hangoutLink },
}),
},
});
}
if (event && event.data.id && event.data.hangoutLink) {
calendar.events.patch({
// Update the same event but this time we know the hangout link
calendarId: selectedCalendar,
auth: myGoogleAuth,
eventId: event.data.id || "",
requestBody: {
description: getRichDescription({
...calEventRaw,
additionalInformation: { hangoutLink: event.data.hangoutLink },
}),
},
});
}
return resolve({
uid: "",
...event.data,
id: event.data.id || "",
additionalInfo: {
hangoutLink: event.data.hangoutLink || "",
},
type: "google_calendar",
password: "",
url: "",
iCalUID: event.data.iCalUID,
});
}
);
});
return {
uid: "",
...event.data,
id: event.data.id || "",
additionalInfo: {
hangoutLink: event.data.hangoutLink || "",
},
type: "google_calendar",
password: "",
url: "",
iCalUID: event.data.iCalUID,
};
} catch (error) {
console.error("There was an error contacting google calendar service: ", error);
throw error;
}
}
async updateEvent(uid: string, event: CalendarEvent, externalCalendarId: string): Promise<any> {
return new Promise(async (resolve, reject) => {
const myGoogleAuth = await this.auth.getToken();
const payload: calendar_v3.Schema$Event = {
summary: event.title,
description: getRichDescription(event),
start: {
dateTime: event.startTime,
timeZone: event.organizer.timeZone,
},
end: {
dateTime: event.endTime,
timeZone: event.organizer.timeZone,
},
attendees: this.getAttendees(event),
reminders: {
useDefault: true,
},
guestsCanSeeOtherGuests: !!event.seatsPerTimeSlot ? event.seatsShowAttendees : true,
};
const payload: calendar_v3.Schema$Event = {
summary: event.title,
description: getRichDescription(event),
start: {
dateTime: event.startTime,
timeZone: event.organizer.timeZone,
},
end: {
dateTime: event.endTime,
timeZone: event.organizer.timeZone,
},
attendees: this.getAttendees(event),
reminders: {
useDefault: true,
},
guestsCanSeeOtherGuests: !!event.seatsPerTimeSlot ? event.seatsShowAttendees : true,
};
if (event.location) {
payload["location"] = getLocation(event);
}
if (event.location) {
payload["location"] = getLocation(event);
}
if (event.conferenceData && event.location === MeetLocationType) {
payload["conferenceData"] = event.conferenceData;
}
if (event.conferenceData && event.location === MeetLocationType) {
payload["conferenceData"] = event.conferenceData;
}
const calendar = google.calendar({
version: "v3",
auth: myGoogleAuth,
const calendar = await this.authedCalendar();
const selectedCalendar = externalCalendarId
? externalCalendarId
: event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId;
try {
const evt = await calendar.events.update({
calendarId: selectedCalendar,
eventId: uid,
sendNotifications: true,
sendUpdates: "none",
requestBody: payload,
conferenceDataVersion: 1,
});
const selectedCalendar = externalCalendarId
? externalCalendarId
: event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId;
calendar.events.update(
{
auth: myGoogleAuth,
if (evt && evt.data.id && evt.data.hangoutLink && event.location === MeetLocationType) {
calendar.events.patch({
// Update the same event but this time we know the hangout link
calendarId: selectedCalendar,
eventId: uid,
sendNotifications: true,
sendUpdates: "none",
requestBody: payload,
conferenceDataVersion: 1,
},
function (err, evt) {
if (err) {
console.error("There was an error contacting google calendar service: ", err);
return reject(err);
}
if (evt && evt.data.id && evt.data.hangoutLink && event.location === MeetLocationType) {
calendar.events.patch({
// Update the same event but this time we know the hangout link
calendarId: selectedCalendar,
auth: myGoogleAuth,
eventId: evt.data.id || "",
requestBody: {
description: getRichDescription({
...event,
additionalInformation: { hangoutLink: evt.data.hangoutLink },
}),
},
});
return resolve({
uid: "",
...evt.data,
id: evt.data.id || "",
additionalInfo: {
hangoutLink: evt.data.hangoutLink || "",
},
type: "google_calendar",
password: "",
url: "",
iCalUID: evt.data.iCalUID,
});
}
return resolve(evt?.data);
}
);
});
eventId: evt.data.id || "",
requestBody: {
description: getRichDescription({
...event,
additionalInformation: { hangoutLink: evt.data.hangoutLink },
}),
},
});
return {
uid: "",
...evt.data,
id: evt.data.id || "",
additionalInfo: {
hangoutLink: evt.data.hangoutLink || "",
},
type: "google_calendar",
password: "",
url: "",
iCalUID: evt.data.iCalUID,
};
}
return evt?.data;
} catch (error) {
console.error("There was an error contacting google calendar service: ", error);
throw error;
}
}
async deleteEvent(uid: string, event: CalendarEvent, externalCalendarId?: string | null): Promise<void> {
return new Promise(async (resolve, reject) => {
const myGoogleAuth = await this.auth.getToken();
const calendar = google.calendar({
version: "v3",
auth: myGoogleAuth,
const calendar = await this.authedCalendar();
const defaultCalendarId = "primary";
const calendarId = externalCalendarId
? externalCalendarId
: event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId;
try {
const event = await calendar.events.delete({
calendarId: calendarId ? calendarId : defaultCalendarId,
eventId: uid,
sendNotifications: false,
sendUpdates: "none",
});
return event?.data;
} catch (error) {
const err = error as GoogleCalError;
/**
* 410 is when an event is already deleted on the Google cal before on cal.com
* 404 is when the event is on a different calendar
*/
if (err.code === 410) return;
console.error("There was an error contacting google calendar service: ", err);
if (err.code === 404) return;
throw err;
}
}
const defaultCalendarId = "primary";
const calendarId = externalCalendarId
? externalCalendarId
: event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId;
calendar.events.delete(
{
auth: myGoogleAuth,
calendarId: calendarId ? calendarId : defaultCalendarId,
eventId: uid,
sendNotifications: false,
sendUpdates: "none",
async getCacheOrFetchAvailability(args: {
timeMin: string;
timeMax: string;
items: { id: string }[];
}): Promise<calendar_v3.Schema$FreeBusyResponse> {
const calendar = await this.authedCalendar();
const flags = await getFeatureFlagMap(prisma);
if (!flags["calendar-cache"]) {
this.log.warn("Calendar Cache is disabled - Skipping");
const { timeMin, timeMax, items } = args;
const apires = await calendar.freebusy.query({
requestBody: { timeMin, timeMax, items },
});
return apires.data;
}
const { timeMin: _timeMin, timeMax: _timeMax, items } = args;
const { timeMin, timeMax } = handleMinMax(_timeMin, _timeMax);
const key = JSON.stringify({ timeMin, timeMax, items });
const cached = await prisma.calendarCache.findUnique({
where: {
credentialId_key: {
credentialId: this.credential.id,
key,
},
function (err: GoogleCalError | null, event) {
if (err) {
/**
* 410 is when an event is already deleted on the Google cal before on cal.com
* 404 is when the event is on a different calendar
*/
if (err.code === 410) return resolve();
console.error("There was an error contacting google calendar service: ", err);
if (err.code === 404) return resolve();
return reject(err);
}
return resolve(event?.data);
}
);
expiresAt: { gte: new Date(Date.now()) },
},
});
if (cached) return cached.value as unknown as calendar_v3.Schema$FreeBusyResponse;
const apires = await calendar.freebusy.query({
requestBody: { timeMin, timeMax, items },
});
// Skipping await to respond faster
prisma.calendarCache.upsert({
where: {
credentialId_key: {
credentialId: this.credential.id,
key,
},
},
update: {
value: JSON.parse(JSON.stringify(apires.data)),
expiresAt: new Date(Date.now() + CACHING_TIME),
},
create: {
value: JSON.parse(JSON.stringify(apires.data)),
credentialId: this.credential.id,
key,
expiresAt: new Date(Date.now() + CACHING_TIME),
},
});
return apires.data;
}
async getAvailability(
@ -339,96 +407,65 @@ export default class GoogleCalendarService implements Calendar {
dateTo: string,
selectedCalendars: IntegrationCalendar[]
): Promise<EventBusyDate[]> {
return new Promise(async (resolve, reject) => {
const myGoogleAuth = await this.auth.getToken();
const calendar = google.calendar({
version: "v3",
auth: myGoogleAuth,
const calendar = await this.authedCalendar();
const selectedCalendarIds = selectedCalendars
.filter((e) => e.integration === this.integrationName)
.map((e) => e.externalId);
if (selectedCalendarIds.length === 0 && selectedCalendars.length > 0) {
// Only calendars of other integrations selected
return [];
}
async function getCalIds() {
if (selectedCalendarIds.length !== 0) return selectedCalendarIds;
const cals = await calendar.calendarList.list({ fields: "items(id)" });
if (!cals.data.items) return [];
return cals.data.items.reduce((c, cal) => (cal.id ? [...c, cal.id] : c), [] as string[]);
}
try {
const calsIds = await getCalIds();
const freeBusyData = await this.getCacheOrFetchAvailability({
timeMin: dateFrom,
timeMax: dateTo,
items: calsIds.map((id) => ({ id })),
});
const selectedCalendarIds = selectedCalendars
.filter((e) => e.integration === this.integrationName)
.map((e) => e.externalId);
if (selectedCalendarIds.length === 0 && selectedCalendars.length > 0) {
// Only calendars of other integrations selected
resolve([]);
return;
}
(selectedCalendarIds.length === 0
? calendar.calendarList
.list({
fields: "items(id)",
})
.then((cals) => cals.data.items?.map((cal) => cal.id).filter(Boolean) || [])
: Promise.resolve(selectedCalendarIds)
)
.then((calsIds) => {
calendar.freebusy.query(
{
requestBody: {
timeMin: dateFrom,
timeMax: dateTo,
items: calsIds.map((id) => ({ id: id })),
},
},
(err, apires) => {
if (err) return reject(err);
// If there's no calendar we just skip
if (!apires?.data.calendars) return resolve([]);
const result = Object.values(apires.data.calendars).reduce((c, i) => {
i.busy?.forEach((busyTime) => {
c.push({
start: busyTime.start || "",
end: busyTime.end || "",
});
});
return c;
}, [] as Prisma.PromiseReturnType<CalendarService["getAvailability"]>);
resolve(result);
}
);
})
.catch((err) => {
this.log.error("There was an error contacting google calendar service: ", err);
reject(err);
if (!freeBusyData?.calendars) throw new Error("No response from google calendar");
const result = Object.values(freeBusyData.calendars).reduce((c, i) => {
i.busy?.forEach((busyTime) => {
c.push({
start: busyTime.start || "",
end: busyTime.end || "",
});
});
});
return c;
}, [] as Prisma.PromiseReturnType<CalendarService["getAvailability"]>);
return result;
} catch (error) {
this.log.error("There was an error contacting google calendar service: ", error);
throw error;
}
}
async listCalendars(): Promise<IntegrationCalendar[]> {
return new Promise(async (resolve, reject) => {
const myGoogleAuth = await this.auth.getToken();
const calendar = google.calendar({
version: "v3",
auth: myGoogleAuth,
});
calendar.calendarList
.list({
fields: "items(id,summary,primary,accessRole)",
})
.then((cals) => {
resolve(
cals.data.items?.map((cal) => {
const calendar: IntegrationCalendar = {
externalId: cal.id ?? "No id",
integration: this.integrationName,
name: cal.summary ?? "No name",
primary: cal.primary ?? false,
readOnly: !(cal.accessRole === "writer" || cal.accessRole === "owner") && true,
email: cal.id ?? "",
};
return calendar;
}) || []
);
})
.catch((err: Error) => {
this.log.error("There was an error contacting google calendar service: ", err);
reject(err);
});
});
const calendar = await this.authedCalendar();
try {
const cals = await calendar.calendarList.list({ fields: "items(id,summary,primary,accessRole)" });
if (!cals.data.items) return [];
return cals.data.items.map(
(cal) =>
({
externalId: cal.id ?? "No id",
integration: this.integrationName,
name: cal.summary ?? "No name",
primary: cal.primary ?? false,
readOnly: !(cal.accessRole === "writer" || cal.accessRole === "owner") && true,
email: cal.id ?? "",
} satisfies IntegrationCalendar)
);
} catch (error) {
this.log.error("There was an error contacting google calendar service: ", error);
throw error;
}
}
}

View File

@ -3,6 +3,7 @@
* Maybe later on we can add string variants or numeric ones
**/
export type AppFlags = {
"calendar-cache": boolean;
emails: boolean;
insights: boolean;
teams: boolean;

View File

@ -0,0 +1,26 @@
-- CreateTable
CREATE TABLE
"CalendarCache" (
"key" TEXT NOT NULL,
"value" JSONB NOT NULL,
"expiresAt" TIMESTAMP(3) NOT NULL,
"credentialId" INTEGER NOT NULL,
CONSTRAINT "CalendarCache_pkey" PRIMARY KEY ("credentialId", "key")
);
-- CreateIndex
CREATE UNIQUE INDEX "CalendarCache_credentialId_key_key" ON "CalendarCache" ("credentialId", "key");
-- AddForeignKey
ALTER TABLE "CalendarCache" ADD CONSTRAINT "CalendarCache_credentialId_fkey" FOREIGN KEY ("credentialId") REFERENCES "Credential" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- Add Feature Flag
INSERT INTO
"Feature" (slug, enabled, description, "type")
VALUES
(
'calendar-cache',
false,
'Enable Third Party Calendar Cache - Cache third party calendar events to reduce the number of API calls to third party calendar providers.',
'OPERATIONAL'
) ON CONFLICT (slug) DO NOTHING;

View File

@ -139,6 +139,7 @@ model Credential {
destinationCalendars DestinationCalendar[]
selectedCalendars SelectedCalendar[]
invalid Boolean? @default(false)
CalendarCache CalendarCache[]
@@index([userId])
@@index([appId])
@ -923,3 +924,15 @@ view BookingTimeStatus {
timeStatus String?
eventParentId Int?
}
model CalendarCache {
// The key would be the unique URL that is requested by the user
key String
value Json
expiresAt DateTime
credentialId Int
credential Credential? @relation(fields: [credentialId], references: [id], onDelete: Cascade)
@@id([credentialId, key])
@@unique([credentialId, key])
}