Fix/insights load speed (#8364)

* Adding skipBatch to trpc calls

* Use view in trpc insights

* Added performance data creation on seed-insights

* fix some scenarios

* Fix migration view fields

* Prevent revalidating too often

---------

Co-authored-by: sean-brydon <55134778+sean-brydon@users.noreply.github.com>
Co-authored-by: Peer Richelsen <peeroke@gmail.com>
Co-authored-by: zomars <zomars@me.com>
This commit is contained in:
alannnc 2023-04-19 13:14:09 -07:00 committed by GitHub
parent d3f1f8b906
commit 0686c08de3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 372 additions and 134 deletions

View File

@ -16,13 +16,21 @@ export const AverageEventDurationChart = () => {
const [startDate, endDate] = dateRange;
const { selectedTeamId: teamId, selectedUserId } = filter;
const { data, isSuccess, isLoading } = trpc.viewer.insights.averageEventDuration.useQuery({
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId: teamId ?? undefined,
memberUserId: selectedMemberUserId ?? undefined,
userId: selectedUserId ?? undefined,
});
const { data, isSuccess, isLoading } = trpc.viewer.insights.averageEventDuration.useQuery(
{
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId: teamId ?? undefined,
memberUserId: selectedMemberUserId ?? undefined,
userId: selectedUserId ?? undefined,
},
{
staleTime: 30000,
trpc: {
context: { skipBatch: true },
},
}
);
if (isLoading) return <LoadingInsight />;

View File

@ -16,14 +16,22 @@ export const BookingKPICards = () => {
const { selectedTeamId: teamId } = filter;
const { data, isSuccess, isLoading } = trpc.viewer.insights.eventsByStatus.useQuery({
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId,
eventTypeId: selectedEventTypeId ?? undefined,
memberUserId: selectedMemberUserId ?? undefined,
userId: selectedUserId ?? undefined,
});
const { data, isSuccess, isLoading } = trpc.viewer.insights.eventsByStatus.useQuery(
{
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId,
eventTypeId: selectedEventTypeId ?? undefined,
memberUserId: selectedMemberUserId ?? undefined,
userId: selectedUserId ?? undefined,
},
{
staleTime: 30000,
trpc: {
context: { skipBatch: true },
},
}
);
const categories: {
title: string;

View File

@ -27,14 +27,22 @@ export const BookingStatusLineChart = () => {
data: eventsTimeLine,
isSuccess,
isLoading,
} = trpc.viewer.insights.eventsTimeline.useQuery({
timeView: selectedTimeView,
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId: selectedTeamId ?? undefined,
eventTypeId: selectedEventTypeId ?? undefined,
userId: selectedUserId ?? undefined,
});
} = trpc.viewer.insights.eventsTimeline.useQuery(
{
timeView: selectedTimeView,
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId: selectedTeamId ?? undefined,
eventTypeId: selectedEventTypeId ?? undefined,
userId: selectedUserId ?? undefined,
},
{
staleTime: 30000,
trpc: {
context: { skipBatch: true },
},
}
);
if (isLoading) return <LoadingInsight />;

View File

@ -14,12 +14,20 @@ export const LeastBookedTeamMembersTable = () => {
const { dateRange, selectedEventTypeId, selectedTeamId: teamId } = filter;
const [startDate, endDate] = dateRange;
const { data, isSuccess, isLoading } = trpc.viewer.insights.membersWithLeastBookings.useQuery({
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId,
eventTypeId: selectedEventTypeId ?? undefined,
});
const { data, isSuccess, isLoading } = trpc.viewer.insights.membersWithLeastBookings.useQuery(
{
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId,
eventTypeId: selectedEventTypeId ?? undefined,
},
{
staleTime: 30000,
trpc: {
context: { skipBatch: true },
},
}
);
if (isLoading) return <LoadingInsight />;

View File

@ -15,12 +15,20 @@ export const MostBookedTeamMembersTable = () => {
const [startDate, endDate] = dateRange;
const { selectedTeamId: teamId } = filter;
const { data, isSuccess, isLoading } = trpc.viewer.insights.membersWithMostBookings.useQuery({
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId,
eventTypeId: selectedEventTypeId ?? undefined,
});
const { data, isSuccess, isLoading } = trpc.viewer.insights.membersWithMostBookings.useQuery(
{
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId,
eventTypeId: selectedEventTypeId ?? undefined,
},
{
staleTime: 30000,
trpc: {
context: { skipBatch: true },
},
}
);
if (isLoading) return <LoadingInsight />;

View File

@ -14,13 +14,21 @@ export const PopularEventsTable = () => {
const [startDate, endDate] = dateRange;
const { selectedTeamId: teamId } = filter;
const { data, isSuccess, isLoading } = trpc.viewer.insights.popularEventTypes.useQuery({
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId: teamId ?? undefined,
userId: selectedUserId ?? undefined,
memberUserId: selectedMemberUserId ?? undefined,
});
const { data, isSuccess, isLoading } = trpc.viewer.insights.popularEventTypes.useQuery(
{
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
teamId: teamId ?? undefined,
userId: selectedUserId ?? undefined,
memberUserId: selectedMemberUserId ?? undefined,
},
{
staleTime: 30000,
trpc: {
context: { skipBatch: true },
},
}
);
if (isLoading) return <LoadingInsight />;

View File

@ -117,9 +117,10 @@ export function FiltersProvider({ children }: { children: React.ReactNode }) {
setSelectedTimeView: (selectedTimeView) => setSelectedTimeView(selectedTimeView),
setSelectedMemberUserId: (selectedMemberUserId) => {
setSelectedMemberUserId(selectedMemberUserId);
const { userId, eventTypeId, ...rest } = router.query;
router.push({
query: {
...router.query,
...rest,
memberUserId: selectedMemberUserId,
},
});
@ -127,8 +128,10 @@ export function FiltersProvider({ children }: { children: React.ReactNode }) {
setSelectedTeamId: (selectedTeamId) => {
setSelectedTeamId(selectedTeamId);
setSelectedUserId(null);
setSelectedMemberUserId(null);
setSelectedEventTypeId(null);
const { teamId, eventTypeId, memberUserId, ...rest } = router.query;
router.replace({
router.push({
query: {
...rest,
teamId: selectedTeamId,
@ -139,8 +142,9 @@ export function FiltersProvider({ children }: { children: React.ReactNode }) {
setSelectedUserId(selectedUserId);
setSelectedTeamId(null);
setSelectedTeamName(null);
setSelectedEventTypeId(null);
const { teamId, eventTypeId, memberUserId, ...rest } = router.query;
router.replace({
router.push({
query: {
...rest,
userId: selectedUserId,

View File

@ -11,10 +11,13 @@ interface ITimeRange {
type TimeViewType = "week" | "month" | "year" | "day";
class EventsInsights {
static getBookingsInTimeRange = async (timeRange: ITimeRange, where: Prisma.BookingWhereInput) => {
static getBookingsInTimeRange = async (
timeRange: ITimeRange,
where: Prisma.BookingTimeStatusWhereInput
) => {
const { start, end } = timeRange;
const events = await prisma.booking.count({
const events = await prisma.bookingTimeStatus.count({
where: {
...where,
createdAt: {
@ -27,48 +30,56 @@ class EventsInsights {
return events;
};
static getCreatedEventsInTimeRange = async (timeRange: ITimeRange, where: Prisma.BookingWhereInput) => {
static getCreatedEventsInTimeRange = async (
timeRange: ITimeRange,
where: Prisma.BookingTimeStatusWhereInput
) => {
const result = await this.getBookingsInTimeRange(timeRange, where);
return result;
};
static getCancelledEventsInTimeRange = async (timeRange: ITimeRange, where: Prisma.BookingWhereInput) => {
static getCancelledEventsInTimeRange = async (
timeRange: ITimeRange,
where: Prisma.BookingTimeStatusWhereInput
) => {
const result = await this.getBookingsInTimeRange(timeRange, {
...where,
status: "CANCELLED",
timeStatus: "cancelled",
});
return result;
};
static getCompletedEventsInTimeRange = async (timeRange: ITimeRange, where: Prisma.BookingWhereInput) => {
static getCompletedEventsInTimeRange = async (
timeRange: ITimeRange,
where: Prisma.BookingTimeStatusWhereInput
) => {
const result = await this.getBookingsInTimeRange(timeRange, {
...where,
status: "ACCEPTED",
endTime: {
lte: dayjs().toISOString(),
},
timeStatus: "completed",
});
return result;
};
static getRescheduledEventsInTimeRange = async (timeRange: ITimeRange, where: Prisma.BookingWhereInput) => {
static getRescheduledEventsInTimeRange = async (
timeRange: ITimeRange,
where: Prisma.BookingTimeStatusWhereInput
) => {
const result = await this.getBookingsInTimeRange(timeRange, {
...where,
rescheduled: true,
timeStatus: "rescheduled",
});
return result;
};
static getBaseBookingForEventStatus = async (where: Prisma.BookingWhereInput) => {
const baseBookings = await prisma.booking.findMany({
static getBaseBookingForEventStatus = async (where: Prisma.BookingTimeStatusWhereInput) => {
const baseBookings = await prisma.bookingTimeStatus.findMany({
where,
select: {
id: true,
eventType: true,
},
});
@ -76,23 +87,23 @@ class EventsInsights {
};
static getTotalRescheduledEvents = async (bookingIds: number[]) => {
return await prisma.booking.count({
return await prisma.bookingTimeStatus.count({
where: {
id: {
in: bookingIds,
},
rescheduled: true,
timeStatus: "rescheduled",
},
});
};
static getTotalCancelledEvents = async (bookingIds: number[]) => {
return await prisma.booking.count({
return await prisma.bookingTimeStatus.count({
where: {
id: {
in: bookingIds,
},
status: "CANCELLED",
timeStatus: "cancelled",
},
});
};

View File

@ -93,7 +93,7 @@ export const insightsRouter = router({
throw new TRPCError({ code: "UNAUTHORIZED" });
}
let whereConditional: Prisma.BookingWhereInput = {};
let whereConditional: Prisma.BookingTimeStatusWhereInput = {};
if (eventTypeId) {
whereConditional["eventTypeId"] = eventTypeId;
@ -102,9 +102,7 @@ export const insightsRouter = router({
whereConditional["userId"] = memberUserId;
}
if (userId) {
whereConditional["eventType"] = {
teamId: null,
};
whereConditional["teamId"] = null;
whereConditional["userId"] = userId;
}
@ -122,14 +120,13 @@ export const insightsRouter = router({
...whereConditional,
OR: [
{
eventType: {
teamId: teamId,
},
teamId,
},
{
userId: {
in: userIdsFromTeam,
},
teamId: null,
},
],
};
@ -161,9 +158,7 @@ export const insightsRouter = router({
gte: lastPeriodStartDate.toDate(),
lte: lastPeriodEndDate.toDate(),
},
eventType: {
teamId: teamId,
},
teamId: teamId,
});
const lastPeriodBaseBookingIds = lastPeriodBaseBookings.map((b) => b.id);
@ -246,7 +241,7 @@ export const insightsRouter = router({
const timeView = inputTimeView;
let whereConditional: Prisma.BookingWhereInput = {};
let whereConditional: Prisma.BookingTimeStatusWhereInput = {};
if (teamId) {
const usersFromTeam = await ctx.prisma.membership.findMany({
@ -262,14 +257,13 @@ export const insightsRouter = router({
whereConditional = {
OR: [
{
eventType: {
teamId,
},
teamId,
},
{
userId: {
in: userIdsFromTeams,
},
teamId: null,
},
],
};
@ -291,9 +285,7 @@ export const insightsRouter = router({
if (selfUserId && !!whereConditional) {
// In this delete we are deleting the teamId filter
whereConditional["userId"] = selfUserId;
whereConditional["eventType"] = {
teamId: null,
};
whereConditional["teamId"] = null;
}
// Get timeline data
@ -382,7 +374,7 @@ export const insightsRouter = router({
return [];
}
let bookingWhere: Prisma.BookingWhereInput = {
let bookingWhere: Prisma.BookingTimeStatusWhereInput = {
createdAt: {
gte: dayjs(startDate).startOf("day").toDate(),
lte: dayjs(endDate).endOf("day").toDate(),
@ -403,14 +395,13 @@ export const insightsRouter = router({
...bookingWhere,
OR: [
{
eventType: {
teamId,
},
teamId,
},
{
userId: {
in: userIdsFromTeams,
},
teamId: null,
},
],
};
@ -419,16 +410,14 @@ export const insightsRouter = router({
if (userId) {
bookingWhere.userId = userId;
// Don't take bookings from any team
bookingWhere.eventType = {
teamId: null,
};
bookingWhere.teamId = null;
}
if (memberUserId) {
bookingWhere.userId = memberUserId;
}
const bookingsFromSelected = await ctx.prisma.booking.groupBy({
const bookingsFromSelected = await ctx.prisma.bookingTimeStatus.groupBy({
by: ["eventTypeId"],
where: bookingWhere,
_count: {
@ -545,14 +534,14 @@ export const insightsRouter = router({
const startDate = dayjs(startDateString);
const endDate = dayjs(endDateString);
let whereConditional: Prisma.BookingWhereInput = {
let whereConditional: Prisma.BookingTimeStatusWhereInput = {
createdAt: {
gte: dayjs(startDate).startOf("day").toDate(),
lte: dayjs(endDate).endOf("day").toDate(),
},
};
if (userId) {
delete whereConditional.eventType;
delete whereConditional.teamId;
whereConditional["userId"] = userId;
}
@ -571,14 +560,13 @@ export const insightsRouter = router({
...whereConditional,
OR: [
{
eventType: {
teamId,
},
teamId,
},
{
userId: {
in: userIdsFromTeams,
},
teamId: null,
},
],
};
@ -587,9 +575,7 @@ export const insightsRouter = router({
if (memberUserId) {
whereConditional = {
userId: memberUserId,
eventType: {
teamId,
},
teamId,
};
}
@ -614,7 +600,10 @@ export const insightsRouter = router({
const startDate = dayjs(date).startOf(startOfEndOf);
const endDate = dayjs(date).endOf(startOfEndOf);
const bookingsInTimeRange = await ctx.prisma.booking.findMany({
const bookingsInTimeRange = await ctx.prisma.bookingTimeStatus.findMany({
select: {
eventLength: true,
},
where: {
...whereConditional,
createdAt: {
@ -622,14 +611,11 @@ export const insightsRouter = router({
lte: endDate.toDate(),
},
},
include: {
eventType: true,
},
});
const avgDuration =
bookingsInTimeRange.reduce((acc, booking) => {
const duration = booking.eventType?.length || 0;
const duration = booking.eventLength || 0;
return acc + duration;
}, 0) / bookingsInTimeRange.length;
@ -654,21 +640,19 @@ export const insightsRouter = router({
return [];
}
const user = ctx.user;
const eventTypeWhere: Prisma.EventTypeWhereInput = {
teamId: teamId,
};
if (eventTypeId) {
eventTypeWhere["id"] = eventTypeId;
}
const bookingWhere: Prisma.BookingWhereInput = {
eventType: eventTypeWhere,
const bookingWhere: Prisma.BookingTimeStatusWhereInput = {
teamId,
createdAt: {
gte: dayjs(startDate).startOf("day").toDate(),
lte: dayjs(endDate).endOf("day").toDate(),
},
};
if (eventTypeId) {
bookingWhere.eventTypeId = eventTypeId;
}
if (teamId) {
const usersFromTeam = await ctx.prisma.membership.findMany({
where: {
@ -679,22 +663,22 @@ export const insightsRouter = router({
},
});
const userIdsFromTeams = usersFromTeam.map((u) => u.userId);
delete bookingWhere.eventType;
delete bookingWhere.eventTypeId;
delete bookingWhere.teamId;
bookingWhere["OR"] = [
{
eventType: {
teamId,
},
teamId,
},
{
userId: {
in: userIdsFromTeams,
},
teamId: null,
},
];
}
const bookingsFromTeam = await ctx.prisma.booking.groupBy({
const bookingsFromTeam = await ctx.prisma.bookingTimeStatus.groupBy({
by: ["userId"],
where: bookingWhere,
_count: {
@ -753,15 +737,9 @@ export const insightsRouter = router({
return [];
}
const user = ctx.user;
const eventTypeWhere: Prisma.EventTypeWhereInput = {
teamId: teamId,
};
if (eventTypeId) {
eventTypeWhere["id"] = eventTypeId;
}
const bookingWhere: Prisma.BookingWhereInput = {
eventType: eventTypeWhere,
const bookingWhere: Prisma.BookingTimeStatusWhereInput = {
eventTypeId,
createdAt: {
gte: dayjs(startDate).startOf("day").toDate(),
lte: dayjs(endDate).endOf("day").toDate(),
@ -778,22 +756,20 @@ export const insightsRouter = router({
},
});
const userIdsFromTeams = usersFromTeam.map((u) => u.userId);
delete bookingWhere.eventType;
bookingWhere["OR"] = [
{
eventType: {
teamId,
},
teamId,
},
{
userId: {
in: userIdsFromTeams,
},
teamId: null,
},
];
}
const bookingsFromTeam = await ctx.prisma.booking.groupBy({
const bookingsFromTeam = await ctx.prisma.bookingTimeStatus.groupBy({
by: ["userId"],
where: bookingWhere,
_count: {

View File

@ -0,0 +1,34 @@
-- View: public.BookingsTimeStatus
-- DROP VIEW public."BookingsTimeStatus";
CREATE OR REPLACE VIEW public."BookingTimeStatus"
AS
SELECT "Booking".id,
"Booking".uid,
"Booking"."eventTypeId",
"Booking".title,
"Booking".description,
"Booking"."startTime",
"Booking"."endTime",
"Booking"."createdAt",
"Booking".location,
"Booking".paid,
"Booking".status,
"Booking".rescheduled,
"Booking"."userId",
"et"."teamId",
"et"."length" as "eventLength",
CASE
WHEN "Booking".rescheduled IS TRUE THEN 'rescheduled'::text
WHEN "Booking".status = 'cancelled'::"BookingStatus" AND "Booking".rescheduled IS FALSE THEN 'cancelled'::text
WHEN "Booking"."endTime" < now() THEN 'completed'::text
WHEN "Booking"."endTime" > now() THEN 'uncompleted'::text
ELSE NULL::text
END AS "timeStatus"
FROM "Booking"
LEFT JOIN "EventType" et ON "Booking"."eventTypeId" = et.id
LEFT JOIN "Membership" mb ON "mb"."userId" = "Booking"."userId";

View File

@ -8,7 +8,7 @@ datasource db {
generator client {
provider = "prisma-client-js"
previewFeatures = []
previewFeatures = ["views"]
}
generator zod {
@ -817,3 +817,22 @@ model SelectedSlots {
@@unique(fields: [userId, slotUtcStartDate, slotUtcEndDate, uid], name: "selectedSlotUnique")
}
view BookingTimeStatus {
id Int @unique
uid String?
eventTypeId Int?
title String?
description String?
startTime DateTime?
endTime DateTime?
createdAt DateTime?
location String?
paid Boolean?
status BookingStatus?
rescheduled Boolean?
userId Int?
teamId Int?
eventLength Int?
timeStatus String?
}

View File

@ -268,7 +268,7 @@ async function main() {
await prisma.booking.createMany({
data: [
...new Array(100)
...new Array(10000)
.fill(0)
.map(() => shuffle({ ...baseBookingSingle }, dayjs().get("y") - 1, singleEvents)),
],
@ -276,7 +276,7 @@ async function main() {
await prisma.booking.createMany({
data: [
...new Array(100)
...new Array(10000)
.fill(0)
.map(() => shuffle({ ...baseBookingSingle }, dayjs().get("y") - 0, singleEvents)),
],
@ -332,7 +332,7 @@ async function main() {
// Create past bookings -2y, -1y, -0y
await prisma.booking.createMany({
data: [
...new Array(100)
...new Array(10000)
.fill(0)
.map(() =>
shuffle({ ...baseBooking }, dayjs().get("y") - 2, teamEvents, [
@ -345,7 +345,7 @@ async function main() {
await prisma.booking.createMany({
data: [
...new Array(100)
...new Array(10000)
.fill(0)
.map(() =>
shuffle({ ...baseBooking }, dayjs().get("y") - 1, teamEvents, [
@ -358,7 +358,7 @@ async function main() {
await prisma.booking.createMany({
data: [
...new Array(100)
...new Array(10000)
.fill(0)
.map(() =>
shuffle({ ...baseBooking }, dayjs().get("y"), teamEvents, [
@ -392,3 +392,149 @@ main()
await prisma.$disconnect();
process.exit(1);
});
/**
* This will create many users in insights teams with bookings 1y in the past
* Should be run after the main function is executed once
*/
async function createPerformanceData() {
const createExtraMembers = false; // Turn ON to be executed
let extraMembersIds;
const insightsTeam = await prisma.team.findFirst({
where: {
slug: "insights-team",
},
});
if (createExtraMembers) {
const insightsRandomUsers: Prisma.UserCreateManyArgs["data"] = [];
const numInsightsUsers = 50; // Change this value to adjust the number of insights users to create
for (let i = 0; i < numInsightsUsers; i++) {
const timestamp = Date.now();
const email = `insightsuser${timestamp}@example.com`;
const insightsUser = {
email,
password: await hashPassword("insightsuser"),
name: `Insights User ${timestamp}`,
username: `insights-user-${timestamp}`,
completedOnboarding: true,
};
insightsRandomUsers.push(insightsUser);
}
await prisma.user.createMany({
data: insightsRandomUsers,
});
// Lets find the ids of the users we just created
extraMembersIds = await prisma.user.findMany({
where: {
email: {
in: insightsRandomUsers.map((user) => user.email),
},
},
select: {
id: true,
},
});
}
if (createExtraMembers) {
if (insightsTeam === null) {
console.log("This should not happen");
throw new Error("Insights team id is undefined or null");
}
await prisma.membership.createMany({
data: extraMembersIds.map((memberId) => ({
teamId: insightsTeam?.id ?? 1,
userId: memberId.id,
role: "MEMBER",
accepted: true,
})),
});
const updateMemberPromises = extraMembersIds.map((memberId) =>
prisma.team.update({
where: {
id: insightsTeam?.id,
},
data: {
members: {
connect: {
userId_teamId: {
userId: memberId.id,
teamId: insightsTeam?.id ?? 1,
},
},
},
},
})
);
await Promise.all(updateMemberPromises);
// Create events for every Member id
const createEvents = extraMembersIds.map((memberId) => ({
title: `Single Event User - ${memberId.id}`,
slug: `single-event-user-${memberId.id}`,
description: `Single Event User - ${memberId.id}`,
length: 30,
userId: memberId.id,
users: {
connect: {
id: memberId.id,
},
},
}));
const createEventPromises = createEvents.map((data) =>
prisma.eventType.create({
data,
})
);
await Promise.all(createEventPromises);
// load the events we just created
const singleEventsCreated = await prisma.eventType.findMany({
where: {
userId: {
in: extraMembersIds.map((memberId) => memberId.id),
},
},
});
// Create bookings for every single event
const baseBooking = {
uid: "demo performance data booking",
title: "Single Event Booking Perf",
description: "Single Event Booking Perf",
startTime: dayjs().toISOString(),
endTime: dayjs().toISOString(),
eventTypeId: singleEventsCreated[0].id,
};
await prisma.booking.createMany({
data: [
...new Array(10000)
.fill(0)
.map(() => shuffle({ ...baseBooking }, dayjs().get("y"), singleEventsCreated)),
],
});
}
}
createPerformanceData()
.then(async () => {
await prisma.$disconnect();
})
.catch(async (e) => {
console.error(e);
await prisma.user.deleteMany({
where: {
username: {
contains: "insights-user-",
},
},
});
await prisma.$disconnect();
process.exit(1);
});