fix: optimize event buffer (#278)

* fix: how we fetch profiles in the buffer

* perf: optimize event buffer

* remove unused file

* fix

* wip

* wip: try groupmq 2

* try simplified event buffer with duration calculation on the fly instead

packages/db/src/services/clients.service.ts

@@ -1,4 +1,4 @@
-import { cacheable, cacheableLru } from '@openpanel/redis';
+import { cacheable } from '@openpanel/redis';
 import type { Client, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 
@@ -34,7 +34,4 @@ export async function getClientById(
   });
 }
 
-export const getClientByIdCached = cacheableLru(getClientById, {
-  maxSize: 1000,
-  ttl: 60 * 5,
-});
+export const getClientByIdCached = cacheable(getClientById, 60 * 5);

packages/db/src/services/event.service.ts

@@ -168,7 +168,6 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
     device: event.device,
     brand: event.brand,
     model: event.model,
-    duration: event.duration,
     path: event.path,
     origin: event.origin,
     referrer: event.referrer,
@@ -216,7 +215,7 @@ export interface IServiceEvent {
   device?: string | undefined;
   brand?: string | undefined;
   model?: string | undefined;
-  duration: number;
+  duration?: number;
   path: string;
   origin: string;
   referrer: string | undefined;
@@ -247,7 +246,7 @@ export interface IServiceEventMinimal {
   browser?: string | undefined;
   device?: string | undefined;
   brand?: string | undefined;
-  duration: number;
+  duration?: number;
   path: string;
   origin: string;
   referrer: string | undefined;
@@ -379,7 +378,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
     device: payload.device ?? '',
     brand: payload.brand ?? '',
     model: payload.model ?? '',
-    duration: payload.duration,
+    duration: payload.duration ?? 0,
     referrer: payload.referrer ?? '',
     referrer_name: payload.referrerName ?? '',
     referrer_type: payload.referrerType ?? '',
@@ -477,7 +476,7 @@ export async function getEventList(options: GetEventListOptions) {
     sb.where.cursor = `created_at < ${sqlstring.escape(formatClickhouseDate(cursor))}`;
   }
 
-  if (!cursor && !(startDate && endDate)) {
+  if (!(cursor || (startDate && endDate))) {
     sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;
   }
 
@@ -562,9 +561,6 @@ export async function getEventList(options: GetEventListOptions) {
   if (select.model) {
     sb.select.model = 'model';
   }
-  if (select.duration) {
-    sb.select.duration = 'duration';
-  }
   if (select.path) {
     sb.select.path = 'path';
   }
@@ -771,7 +767,6 @@ class EventService {
     where,
     select,
     limit,
-    orderBy,
     filters,
   }: {
     projectId: string;
@@ -811,7 +806,6 @@ class EventService {
         select.event.deviceId && 'e.device_id as device_id',
         select.event.name && 'e.name as name',
         select.event.path && 'e.path as path',
-        select.event.duration && 'e.duration as duration',
         select.event.country && 'e.country as country',
         select.event.city && 'e.city as city',
         select.event.os && 'e.os as os',
@@ -896,7 +890,6 @@ class EventService {
         select.event.deviceId && 'e.device_id as device_id',
         select.event.name && 'e.name as name',
         select.event.path && 'e.path as path',
-        select.event.duration && 'e.duration as duration',
         select.event.country && 'e.country as country',
         select.event.city && 'e.city as city',
         select.event.os && 'e.os as os',
@@ -1032,7 +1025,6 @@ class EventService {
           id: true,
           name: true,
           createdAt: true,
-          duration: true,
           country: true,
           city: true,
           os: true,

packages/db/src/services/notification.service.ts

@@ -90,7 +90,7 @@ export const getNotificationRulesByProjectId = cacheable(
       },
     });
   },
-  60 * 24
+  60 * 24,
 );
 
 function getIntegration(integrationId: string | null) {

packages/db/src/services/overview.service.ts

@@ -416,6 +416,30 @@ export class OverviewService {
     const where = this.getRawWhereClause('sessions', filters);
     const fillConfig = this.getFillConfig(interval, startDate, endDate);
 
+    // CTE: per-event screen_view durations via window function
+    const rawScreenViewDurationsQuery = clix(this.client, timezone)
+      .select([
+        `${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
+        `dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
+      ])
+      .from(TABLE_NAMES.events)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'screen_view')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ])
+      .rawWhere(this.getRawWhereClause('events', filters));
+
+    // CTE: avg duration per date bucket
+    const avgDurationByDateQuery = clix(this.client, timezone)
+      .select([
+        'date',
+        'round(avgIf(duration, duration > 0), 2) / 1000 AS avg_session_duration',
+      ])
+      .from('raw_screen_view_durations')
+      .groupBy(['date']);
+
     // Session aggregation with bounce rates
     const sessionAggQuery = clix(this.client, timezone)
       .select([
@@ -473,6 +497,8 @@ export class OverviewService {
           .where('date', '!=', rollupDate)
       )
       .with('overall_unique_visitors', overallUniqueVisitorsQuery)
+      .with('raw_screen_view_durations', rawScreenViewDurationsQuery)
+      .with('avg_duration_by_date', avgDurationByDateQuery)
       .select<{
         date: string;
         bounce_rate: number;
@@ -489,8 +515,7 @@ export class OverviewService {
         'dss.bounce_rate as bounce_rate',
         'uniq(e.profile_id) AS unique_visitors',
         'uniq(e.session_id) AS total_sessions',
-        'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
-        'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
+        'coalesce(dur.avg_session_duration, 0) AS avg_session_duration',
         'count(*) AS total_screen_views',
         'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
         '(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
@@ -502,6 +527,10 @@ export class OverviewService {
         'daily_session_stats AS dss',
         `${clix.toStartOf('e.created_at', interval as any)} = dss.date`
       )
+      .leftJoin(
+        'avg_duration_by_date AS dur',
+        `${clix.toStartOf('e.created_at', interval as any)} = dur.date`
+      )
       .where('e.project_id', '=', projectId)
       .where('e.name', '=', 'screen_view')
       .where('e.created_at', 'BETWEEN', [
@@ -509,7 +538,7 @@ export class OverviewService {
         clix.datetime(endDate, 'toDateTime'),
       ])
       .rawWhere(this.getRawWhereClause('events', filters))
-      .groupBy(['date', 'dss.bounce_rate'])
+      .groupBy(['date', 'dss.bounce_rate', 'dur.avg_session_duration'])
       .orderBy('date', 'ASC')
       .fill(fillConfig.from, fillConfig.to, fillConfig.step)
       .transform({

packages/db/src/services/pages.service.ts

@@ -52,6 +52,24 @@ export class PagesService {
       .where('created_at', '>=', clix.exp('now() - INTERVAL 30 DAY'))
       .groupBy(['origin', 'path']);
 
+    // CTE: compute screen_view durations via window function (leadInFrame gives next event's timestamp)
+    const screenViewDurationsCte = clix(this.client, timezone)
+      .select([
+        'project_id',
+        'session_id',
+        'path',
+        'origin',
+        `dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
+      ])
+      .from(TABLE_NAMES.events, false)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'screen_view')
+      .where('path', '!=', '')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ]);
+
     // Pre-filtered sessions subquery for better performance
     const sessionsSubquery = clix(this.client, timezone)
       .select(['id', 'project_id', 'is_bounce'])
@@ -66,6 +84,7 @@ export class PagesService {
     // Main query: aggregate events and calculate bounce rate from pre-filtered sessions
     const query = clix(this.client, timezone)
       .with('page_titles', titlesCte)
+      .with('screen_view_durations', screenViewDurationsCte)
       .select<ITopPage>([
         'e.origin as origin',
         'e.path as path',
@@ -74,25 +93,18 @@ export class PagesService {
         'count() as pageviews',
         'round(avg(e.duration) / 1000 / 60, 2) as avg_duration',
         `round(
-          (uniqIf(e.session_id, s.is_bounce = 1) * 100.0) / 
-          nullIf(uniq(e.session_id), 0), 
+          (uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /
+          nullIf(uniq(e.session_id), 0),
           2
         ) as bounce_rate`,
       ])
-      .from(`${TABLE_NAMES.events} e`, false)
+      .from('screen_view_durations e', false)
       .leftJoin(
         sessionsSubquery,
         'e.session_id = s.id AND e.project_id = s.project_id',
         's'
       )
       .leftJoin('page_titles pt', 'concat(e.origin, e.path) = pt.page_key')
-      .where('e.project_id', '=', projectId)
-      .where('e.name', '=', 'screen_view')
-      .where('e.path', '!=', '')
-      .where('e.created_at', 'BETWEEN', [
-        clix.datetime(startDate, 'toDateTime'),
-        clix.datetime(endDate, 'toDateTime'),
-      ])
       .when(!!search, (q) => {
         const term = `%${search}%`;
         q.whereGroup()

packages/db/src/services/project.service.ts

@@ -1,6 +1,6 @@
 import { cacheable } from '@openpanel/redis';
 import sqlstring from 'sqlstring';
-import { TABLE_NAMES, chQuery } from '../clickhouse/client';
+import { chQuery, TABLE_NAMES } from '../clickhouse/client';
 import type { Prisma, Project } from '../prisma-client';
 import { db } from '../prisma-client';
 
@@ -25,6 +25,7 @@ export async function getProjectById(id: string) {
   return res;
 }
 
+/** L1 LRU (60s) + L2 Redis. clear() invalidates Redis + local LRU; other nodes may serve stale from LRU for up to 60s. */
 export const getProjectByIdCached = cacheable(getProjectById, 60 * 60 * 24);
 
 export async function getProjectWithClients(id: string) {
@@ -44,7 +45,7 @@ export async function getProjectWithClients(id: string) {
   return res;
 }
 
-export async function getProjectsByOrganizationId(organizationId: string) {
+export function getProjectsByOrganizationId(organizationId: string) {
   return db.project.findMany({
     where: {
       organizationId,
@@ -95,7 +96,7 @@ export async function getProjects({
 
   if (access.length > 0) {
     return projects.filter((project) =>
-      access.some((a) => a.projectId === project.id),
+      access.some((a) => a.projectId === project.id)
     );
   }
 
@@ -104,7 +105,7 @@ export async function getProjects({
 
 export const getProjectEventsCount = async (projectId: string) => {
   const res = await chQuery<{ count: number }>(
-    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
+    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
   );
   return res[0]?.count;
 };

packages/db/src/services/salt.service.ts

@@ -1,9 +1,9 @@
 import { generateSalt } from '@openpanel/common/server';
 
-import { cacheableLru } from '@openpanel/redis';
+import { cacheable } from '@openpanel/redis';
 import { db } from '../prisma-client';
 
-export const getSalts = cacheableLru(
+export const getSalts = cacheable(
   'op:salt',
   async () => {
     const [curr, prev] = await db.salt.findMany({
@@ -24,10 +24,7 @@ export const getSalts = cacheableLru(
 
     return salts;
   },
-  {
-    maxSize: 2,
-    ttl: 60 * 5,
-  },
+  60 * 5,
 );
 
 export async function createInitialSalts() {