feature(dashboard): refactor overview

fix(lint)

packages/trpc/src/root.ts

@@ -7,6 +7,7 @@ import { integrationRouter } from './routers/integration';
 import { notificationRouter } from './routers/notification';
 import { onboardingRouter } from './routers/onboarding';
 import { organizationRouter } from './routers/organization';
+import { overviewRouter } from './routers/overview';
 import { profileRouter } from './routers/profile';
 import { projectRouter } from './routers/project';
 import { referenceRouter } from './routers/reference';
@@ -39,6 +40,7 @@ export const appRouter = createTRPCRouter({
   integration: integrationRouter,
   auth: authRouter,
   subscription: subscriptionRouter,
+  overview: overviewRouter,
 });
 
 // export type definition of API

packages/trpc/src/routers/chart.helpers.ts

@@ -291,12 +291,11 @@ export function getChartPrevStartEndDate({
 }: {
   startDate: string;
   endDate: string;
-  range: IChartRange;
 }) {
   const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate));
   return {
-    startDate: formatISO(subMilliseconds(new Date(startDate), diff - 1)),
-    endDate: formatISO(subMilliseconds(new Date(endDate), diff - 1)),
+    startDate: formatISO(subMilliseconds(new Date(startDate), diff + 1000)),
+    endDate: formatISO(subMilliseconds(new Date(endDate), diff + 1000)),
   };
 }
 
@@ -307,7 +306,10 @@ export async function getFunnelData({
   ...payload
 }: IChartInput) {
   const funnelWindow = (payload.funnelWindow || 24) * 3600;
-  const funnelGroup = payload.funnelGroup || 'session_id';
+  const funnelGroup =
+    payload.funnelGroup === 'profile_id'
+      ? [`COALESCE(nullIf(s.profile_id, ''), e.profile_id)`, 'profile_id']
+      : ['session_id', 'session_id'];
 
   if (!startDate || !endDate) {
     throw new Error('startDate and endDate are required');
@@ -327,16 +329,19 @@ export async function getFunnelData({
     return getWhere().replace('WHERE ', '');
   });
 
-  const innerSql = `SELECT
-    ${funnelGroup},
-    windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
-  FROM ${TABLE_NAMES.events}
-  WHERE 
-    project_id = ${escape(projectId)} AND 
+  const commonWhere = `project_id = ${escape(projectId)} AND 
     created_at >= '${formatClickhouseDate(startDate)}' AND 
-    created_at <= '${formatClickhouseDate(endDate)}' AND
+    created_at <= '${formatClickhouseDate(endDate)}'`;
+
+  const innerSql = `SELECT
+    ${funnelGroup[0]} AS ${funnelGroup[1]},
+    windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
+  FROM ${TABLE_NAMES.events} e
+  ${funnelGroup[0] === 'session_id' ? '' : `LEFT JOIN (SELECT profile_id, id FROM sessions WHERE ${commonWhere}) AS s ON s.id = e.session_id`}
+  WHERE 
+    ${commonWhere} AND
     name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
-  GROUP BY ${funnelGroup}`;
+  GROUP BY ${funnelGroup[0]}`;
 
   const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;
 
@@ -513,10 +518,7 @@ export async function getChart(input: IChartInput) {
   }
 
   const currentPeriod = getChartStartEndDate(input);
-  const previousPeriod = getChartPrevStartEndDate({
-    range: input.range,
-    ...currentPeriod,
-  });
+  const previousPeriod = getChartPrevStartEndDate(currentPeriod);
 
   // If the current period end date is after the subscription chart end date, we need to use the subscription chart end date
   if (

packages/trpc/src/routers/chart.ts

@@ -181,10 +181,7 @@ export const chartRouter = createTRPCRouter({
 
   funnel: protectedProcedure.input(zChartInput).query(async ({ input }) => {
     const currentPeriod = getChartStartEndDate(input);
-    const previousPeriod = getChartPrevStartEndDate({
-      range: input.range,
-      ...currentPeriod,
-    });
+    const previousPeriod = getChartPrevStartEndDate(currentPeriod);
 
     const [current, previous] = await Promise.all([
       getFunnelData({ ...input, ...currentPeriod }),

packages/trpc/src/routers/event.ts

@@ -3,16 +3,21 @@ import { escape } from 'sqlstring';
 import { z } from 'zod';
 
 import {
+  type IServiceProfile,
   TABLE_NAMES,
   chQuery,
   convertClickhouseDateToJs,
   db,
+  eventService,
+  formatClickhouseDate,
   getEventList,
   getEvents,
   getTopPages,
 } from '@openpanel/db';
 import { zChartEventFilter } from '@openpanel/validation';
 
+import { addMinutes, subMinutes } from 'date-fns';
+import { clone } from 'ramda';
 import { getProjectAccessCached } from '../access';
 import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
@@ -48,51 +53,83 @@ export const eventRouter = createTRPCRouter({
       z.object({
         id: z.string(),
         projectId: z.string(),
+        createdAt: z.date().optional(),
       }),
     )
-    .query(async ({ input: { id, projectId } }) => {
-      const res = await getEvents(
-        `SELECT * FROM ${TABLE_NAMES.events} WHERE id = ${escape(id)} AND project_id = ${escape(projectId)};`,
-        {
-          meta: true,
-        },
-      );
+    .query(async ({ input: { id, projectId, createdAt } }) => {
+      const res = await eventService.getById({
+        projectId,
+        id,
+        createdAt,
+      });
 
-      if (!res?.[0]) {
+      if (!res) {
         throw new TRPCError({
           code: 'NOT_FOUND',
           message: 'Event not found',
         });
       }
 
-      return res[0];
+      return res;
     }),
 
   events: protectedProcedure
     .input(
       z.object({
         projectId: z.string(),
-        cursor: z.number().optional(),
         profileId: z.string().optional(),
-        take: z.number().default(50),
-        events: z.array(z.string()).optional(),
+        cursor: z.string().optional(),
         filters: z.array(zChartEventFilter).default([]),
         startDate: z.date().optional(),
         endDate: z.date().optional(),
-        meta: z.boolean().optional(),
-        profile: z.boolean().optional(),
       }),
     )
     .query(async ({ input }) => {
-      return getEventList(input);
+      const items = await getEventList({
+        ...input,
+        take: 50,
+        cursor: input.cursor ? new Date(input.cursor) : undefined,
+      });
+
+      // Hacky join to get profile for entire session
+      // TODO: Replace this with a join on the session table
+      const map = new Map<string, IServiceProfile>(); // sessionId -> profileId
+      for (const item of items) {
+        if (item.sessionId && item.profile?.isExternal === true) {
+          map.set(item.sessionId, item.profile);
+        }
+      }
+
+      for (const item of items) {
+        const profile = map.get(item.sessionId);
+        if (profile && (item.profile?.isExternal === false || !item.profile)) {
+          item.profile = clone(profile);
+          if (item?.profile?.firstName) {
+            item.profile.firstName = `* ${item.profile.firstName}`;
+          }
+        }
+      }
+
+      const lastItem = items[items.length - 1];
+
+      return {
+        items,
+        meta: {
+          next:
+            items.length === 50 && lastItem
+              ? lastItem.createdAt.toISOString()
+              : null,
+        },
+      };
     }),
   conversions: protectedProcedure
     .input(
       z.object({
         projectId: z.string(),
+        cursor: z.string().optional(),
       }),
     )
-    .query(async ({ input: { projectId } }) => {
+    .query(async ({ input: { projectId, cursor } }) => {
       const conversions = await db.eventMeta.findMany({
         where: {
           projectId,
@@ -101,16 +138,30 @@ export const eventRouter = createTRPCRouter({
       });
 
       if (conversions.length === 0) {
-        return [];
+        return {
+          items: [],
+          meta: {
+            next: null,
+          },
+        };
       }
 
-      return getEvents(
-        `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,
+      const items = await getEvents(
+        `SELECT * FROM ${TABLE_NAMES.events} WHERE ${cursor ? `created_at <= '${formatClickhouseDate(cursor)}' AND` : ''} project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY toDate(created_at) DESC, created_at DESC LIMIT 50;`,
         {
           profile: true,
           meta: true,
         },
       );
+
+      const lastItem = items[items.length - 1];
+
+      return {
+        items,
+        meta: {
+          next: lastItem ? lastItem.createdAt.toISOString() : null,
+        },
+      };
     }),
 
   bots: publicProcedure

packages/trpc/src/routers/overview.ts

@@ -0,0 +1,156 @@
+import {
+  overviewService,
+  zGetMetricsInput,
+  zGetTopGenericInput,
+  zGetTopPagesInput,
+} from '@openpanel/db';
+import { type IChartRange, zRange } from '@openpanel/validation';
+import { z } from 'zod';
+import { cacheMiddleware, createTRPCRouter, publicProcedure } from '../trpc';
+import {
+  getChartPrevStartEndDate,
+  getChartStartEndDate,
+} from './chart.helpers';
+
+const cacher = cacheMiddleware((input) => {
+  const range = input.range as IChartRange;
+  switch (range) {
+    case '30min':
+    case 'today':
+    case 'lastHour':
+      return 1;
+    default:
+      return 1;
+  }
+});
+
+function getCurrentAndPrevious<
+  T extends {
+    startDate?: string | null;
+    endDate?: string | null;
+    range: IChartRange;
+  },
+>(input: T, fetchPrevious = false) {
+  const current = getChartStartEndDate(input);
+  const previous = getChartPrevStartEndDate(current);
+
+  return async <R>(
+    fn: (input: T & { startDate: string; endDate: string }) => Promise<R>,
+  ): Promise<{
+    current: R;
+    previous: R | null;
+  }> => {
+    const res = await Promise.all([
+      fn({
+        ...input,
+        startDate: current.startDate,
+        endDate: current.endDate,
+      }),
+      fetchPrevious
+        ? fn({
+            ...input,
+            startDate: previous.startDate,
+            endDate: previous.endDate,
+          })
+        : Promise.resolve(null),
+    ]);
+
+    return {
+      current: res[0],
+      previous: res[1],
+    };
+  };
+}
+
+export const overviewRouter = createTRPCRouter({
+  stats: publicProcedure
+    .input(
+      zGetMetricsInput.omit({ startDate: true, endDate: true }).extend({
+        startDate: z.string().nullish(),
+        endDate: z.string().nullish(),
+        range: zRange,
+      }),
+    )
+    .use(cacher)
+    .query(async ({ ctx, input }) => {
+      const { current, previous } = await getCurrentAndPrevious(
+        input,
+        true,
+      )(overviewService.getMetrics.bind(overviewService));
+
+      return {
+        metrics: {
+          ...current.metrics,
+          prev_bounce_rate: previous?.metrics.bounce_rate || null,
+          prev_unique_visitors: previous?.metrics.unique_visitors || null,
+          prev_total_screen_views: previous?.metrics.total_screen_views || null,
+          prev_avg_session_duration:
+            previous?.metrics.avg_session_duration || null,
+          prev_views_per_session: previous?.metrics.views_per_session || null,
+          prev_total_sessions: previous?.metrics.total_sessions || null,
+        },
+        series: current.series.map((item) => {
+          const prev = previous?.series.find((p) => p.date === item.date);
+          return {
+            ...item,
+            prev_bounce_rate: prev?.bounce_rate,
+            prev_unique_visitors: prev?.unique_visitors,
+            prev_total_screen_views: prev?.total_screen_views,
+            prev_avg_session_duration: prev?.avg_session_duration,
+            prev_views_per_session: prev?.views_per_session,
+            prev_total_sessions: prev?.total_sessions,
+          };
+        }),
+      };
+    }),
+
+  topPages: publicProcedure
+    .input(
+      zGetTopPagesInput.omit({ startDate: true, endDate: true }).extend({
+        startDate: z.string().nullish(),
+        endDate: z.string().nullish(),
+        range: zRange,
+        mode: z.enum(['page', 'entry', 'exit', 'bot']),
+      }),
+    )
+    .use(cacher)
+    .query(async ({ input }) => {
+      const { current } = await getCurrentAndPrevious(
+        input,
+        false,
+      )(async (input) => {
+        if (input.mode === 'page') {
+          return overviewService.getTopPages(input);
+        }
+
+        if (input.mode === 'bot') {
+          return Promise.resolve([]);
+        }
+
+        return overviewService.getTopEntryExit({
+          ...input,
+          mode: input.mode,
+        });
+      });
+
+      return current;
+    }),
+
+  topGeneric: publicProcedure
+    .input(
+      zGetTopGenericInput.omit({ startDate: true, endDate: true }).extend({
+        startDate: z.string().nullish(),
+        endDate: z.string().nullish(),
+        range: zRange,
+      }),
+    )
+    .use(cacher)
+    .query(async ({ input }) => {
+      const { current } = await getCurrentAndPrevious(
+        input,
+        false,
+      )(overviewService.getTopGeneric.bind(overviewService));
+
+      return current;
+    }),
+});

packages/trpc/src/trpc.ts

@@ -140,3 +140,39 @@ export const protectedProcedure = t.procedure
   .use(enforceUserIsAuthed)
   .use(enforceAccess)
   .use(loggerMiddleware);
+
+const middlewareMarker = 'middlewareMarker' as 'middlewareMarker' & {
+  __brand: 'middlewareMarker';
+};
+
+export const cacheMiddleware = (cbOrTtl: number | ((input: any) => number)) =>
+  t.middleware(async ({ ctx, next, path, type, rawInput, input }) => {
+    if (type !== 'query') {
+      return next();
+    }
+    let key = `trpc:${path}:`;
+    if (rawInput) {
+      key += JSON.stringify(rawInput).replace(/\"/g, "'");
+    }
+    const cache = await getRedisCache().getJson(key);
+    if (cache) {
+      return {
+        ok: true,
+        data: cache,
+        ctx,
+        marker: middlewareMarker,
+      };
+    }
+    const result = await next();
+
+    // @ts-expect-error
+    if (result.data) {
+      getRedisCache().setJson(
+        key,
+        typeof cbOrTtl === 'function' ? cbOrTtl(input) : cbOrTtl,
+        // @ts-expect-error
+        result.data,
+      );
+    }
+    return result;
+  });