fix: optimize event buffer (#278)

* fix: how we fetch profiles in the buffer

* perf: optimize event buffer

* remove unused file

* fix

* wip

* wip: try groupmq 2

* try simplified event buffer with duration calculation on the fly instead

apps/api/package.json

@@ -30,7 +30,6 @@
     "@openpanel/logger": "workspace:*",
     "@openpanel/payments": "workspace:*",
     "@openpanel/queue": "workspace:*",
-    "groupmq": "catalog:",
     "@openpanel/redis": "workspace:*",
     "@openpanel/trpc": "workspace:*",
     "@openpanel/validation": "workspace:*",
@@ -40,6 +39,7 @@
     "fastify": "^5.6.1",
     "fastify-metrics": "^12.1.0",
     "fastify-raw-body": "^5.0.0",
+    "groupmq": "catalog:",
     "jsonwebtoken": "^9.0.2",
     "ramda": "^0.29.1",
     "sharp": "^0.33.5",

apps/api/src/bots/index.ts

@@ -1,4 +1,4 @@
-import { cacheable, cacheableLru } from '@openpanel/redis';
+import { cacheable } from '@openpanel/redis';
 import bots from './bots';
 
 // Pre-compile regex patterns at module load time
@@ -15,7 +15,7 @@ const compiledBots = bots.map((bot) => {
 const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
 const includesBots = compiledBots.filter((bot) => 'includes' in bot);
 
-export const isBot = cacheableLru(
+export const isBot = cacheable(
   'is-bot',
   (ua: string) => {
     // Check simple string patterns first (fast)
@@ -40,8 +40,5 @@ export const isBot = cacheableLru(
 
     return null;
   },
-  {
+  60 * 5
-    maxSize: 1000,
-    ttl: 60 * 5,
-  },
 );

apps/api/src/controllers/live.controller.ts

@@ -1,12 +1,5 @@
-import type { FastifyRequest } from 'fastify';
-import superjson from 'superjson';
-
 import type { WebSocket } from '@fastify/websocket';
-import {
+import { eventBuffer } from '@openpanel/db';
-  eventBuffer,
-  getProfileById,
-  transformMinimalEvent,
-} from '@openpanel/db';
 import { setSuperJson } from '@openpanel/json';
 import {
   psubscribeToPublishedEvent,
@@ -14,10 +7,7 @@ import {
 } from '@openpanel/redis';
 import { getProjectAccess } from '@openpanel/trpc';
 import { getOrganizationAccess } from '@openpanel/trpc/src/access';
-
+import type { FastifyRequest } from 'fastify';
-export function getLiveEventInfo(key: string) {
-  return key.split(':').slice(2) as [string, string];
-}
 
 export function wsVisitors(
   socket: WebSocket,
@@ -25,27 +15,38 @@ export function wsVisitors(
     Params: {
       projectId: string;
     };
-  }>,
+  }>
 ) {
   const { params } = req;
-  const unsubscribe = subscribeToPublishedEvent('events', 'saved', (event) => {
+  const sendCount = () => {
-    if (event?.projectId === params.projectId) {
+    eventBuffer
-      eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
+      .getActiveVisitorCount(params.projectId)
+      .then((count) => {
         socket.send(String(count));
+      })
+      .catch(() => {
+        socket.send('0');
       });
+  };
+
+  const unsubscribe = subscribeToPublishedEvent(
+    'events',
+    'batch',
+    ({ projectId }) => {
+      if (projectId === params.projectId) {
+        sendCount();
+      }
     }
-  });
+  );
 
   const punsubscribe = psubscribeToPublishedEvent(
     '__keyevent@0__:expired',
     (key) => {
-      const [projectId] = getLiveEventInfo(key);
+      const [, , projectId] = key.split(':');
-      if (projectId && projectId === params.projectId) {
+      if (projectId === params.projectId) {
-        eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
+        sendCount();
-          socket.send(String(count));
-        });
       }
-    },
+    }
   );
 
   socket.on('close', () => {
@@ -62,18 +63,10 @@ export async function wsProjectEvents(
     };
     Querystring: {
       token?: string;
-      type?: 'saved' | 'received';
     };
-  }>,
+  }>
 ) {
-  const { params, query } = req;
+  const { params } = req;
-  const type = query.type || 'saved';
-
-  if (!['saved', 'received'].includes(type)) {
-    socket.send('Invalid type');
-    socket.close();
-    return;
-  }
 
   const userId = req.session?.userId;
   if (!userId) {
@@ -87,24 +80,20 @@ export async function wsProjectEvents(
     projectId: params.projectId,
   });
 
+  if (!access) {
+    socket.send('No access');
+    socket.close();
+    return;
+  }
+
   const unsubscribe = subscribeToPublishedEvent(
     'events',
-    type,
+    'batch',
-    async (event) => {
+    ({ projectId, count }) => {
-      if (event.projectId === params.projectId) {
+      if (projectId === params.projectId) {
-        const profile = await getProfileById(event.profileId, event.projectId);
+        socket.send(setSuperJson({ count }));
-        socket.send(
-          superjson.stringify(
-            access
-              ? {
-                  ...event,
-                  profile,
-                }
-              : transformMinimalEvent(event),
-          ),
-        );
       }
-    },
+    }
   );
 
   socket.on('close', () => unsubscribe());
@@ -116,7 +105,7 @@ export async function wsProjectNotifications(
     Params: {
       projectId: string;
     };
-  }>,
+  }>
 ) {
   const { params } = req;
   const userId = req.session?.userId;
@@ -143,9 +132,9 @@ export async function wsProjectNotifications(
     'created',
     (notification) => {
       if (notification.projectId === params.projectId) {
-        socket.send(superjson.stringify(notification));
+        socket.send(setSuperJson(notification));
       }
-    },
+    }
   );
 
   socket.on('close', () => unsubscribe());
@@ -157,7 +146,7 @@ export async function wsOrganizationEvents(
     Params: {
       organizationId: string;
     };
-  }>,
+  }>
 ) {
   const { params } = req;
   const userId = req.session?.userId;
@@ -184,7 +173,7 @@ export async function wsOrganizationEvents(
     'subscription_updated',
     (message) => {
       socket.send(setSuperJson(message));
-    },
+    }
   );
 
   socket.on('close', () => unsubscribe());

apps/api/src/controllers/manage.controller.ts

@@ -1,5 +1,4 @@
 import crypto from 'node:crypto';
-import { HttpError } from '@/utils/errors';
 import { stripTrailingSlash } from '@openpanel/common';
 import { hashPassword } from '@openpanel/common/server';
 import {
@@ -10,6 +9,7 @@ import {
 } from '@openpanel/db';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { z } from 'zod';
+import { HttpError } from '@/utils/errors';
 
 // Validation schemas
 const zCreateProject = z.object({
@@ -57,7 +57,7 @@ const zUpdateReference = z.object({
 // Projects CRUD
 export async function listProjects(
   request: FastifyRequest,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const projects = await db.project.findMany({
     where: {
@@ -74,7 +74,7 @@ export async function listProjects(
 
 export async function getProject(
   request: FastifyRequest<{ Params: { id: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const project = await db.project.findFirst({
     where: {
@@ -92,7 +92,7 @@ export async function getProject(
 
 export async function createProject(
   request: FastifyRequest<{ Body: z.infer<typeof zCreateProject> }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const parsed = zCreateProject.safeParse(request.body);
 
@@ -139,12 +139,9 @@ export async function createProject(
     },
   });
 
-  // Clear cache
   await Promise.all([
     getProjectByIdCached.clear(project.id),
-    project.clients.map((client) => {
+    ...project.clients.map((client) => getClientByIdCached.clear(client.id)),
-      getClientByIdCached.clear(client.id);
-    }),
   ]);
 
   reply.send({
@@ -165,7 +162,7 @@ export async function updateProject(
     Params: { id: string };
     Body: z.infer<typeof zUpdateProject>;
   }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const parsed = zUpdateProject.safeParse(request.body);
 
@@ -223,12 +220,9 @@ export async function updateProject(
     data: updateData,
   });
 
-  // Clear cache
   await Promise.all([
     getProjectByIdCached.clear(project.id),
-    existing.clients.map((client) => {
+    ...existing.clients.map((client) => getClientByIdCached.clear(client.id)),
-      getClientByIdCached.clear(client.id);
-    }),
   ]);
 
   reply.send({ data: project });
@@ -236,7 +230,7 @@ export async function updateProject(
 
 export async function deleteProject(
   request: FastifyRequest<{ Params: { id: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const project = await db.project.findFirst({
     where: {
@@ -266,7 +260,7 @@ export async function deleteProject(
 // Clients CRUD
 export async function listClients(
   request: FastifyRequest<{ Querystring: { projectId?: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const where: any = {
     organizationId: request.client!.organizationId,
@@ -300,7 +294,7 @@ export async function listClients(
 
 export async function getClient(
   request: FastifyRequest<{ Params: { id: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const client = await db.client.findFirst({
     where: {
@@ -318,7 +312,7 @@ export async function getClient(
 
 export async function createClient(
   request: FastifyRequest<{ Body: z.infer<typeof zCreateClient> }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const parsed = zCreateClient.safeParse(request.body);
 
@@ -374,7 +368,7 @@ export async function updateClient(
     Params: { id: string };
     Body: z.infer<typeof zUpdateClient>;
   }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const parsed = zUpdateClient.safeParse(request.body);
 
@@ -417,7 +411,7 @@ export async function updateClient(
 
 export async function deleteClient(
   request: FastifyRequest<{ Params: { id: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const client = await db.client.findFirst({
     where: {
@@ -444,7 +438,7 @@ export async function deleteClient(
 // References CRUD
 export async function listReferences(
   request: FastifyRequest<{ Querystring: { projectId?: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const where: any = {};
 
@@ -488,7 +482,7 @@ export async function listReferences(
 
 export async function getReference(
   request: FastifyRequest<{ Params: { id: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const reference = await db.reference.findUnique({
     where: {
@@ -516,7 +510,7 @@ export async function getReference(
 
 export async function createReference(
   request: FastifyRequest<{ Body: z.infer<typeof zCreateReference> }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const parsed = zCreateReference.safeParse(request.body);
 
@@ -559,7 +553,7 @@ export async function updateReference(
     Params: { id: string };
     Body: z.infer<typeof zUpdateReference>;
   }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const parsed = zUpdateReference.safeParse(request.body);
 
@@ -616,7 +610,7 @@ export async function updateReference(
 
 export async function deleteReference(
   request: FastifyRequest<{ Params: { id: string } }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const reference = await db.reference.findUnique({
     where: {

apps/api/src/controllers/track.controller.ts

@@ -7,7 +7,10 @@ import {
   upsertProfile,
 } from '@openpanel/db';
 import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
-import { getEventsGroupQueueShard } from '@openpanel/queue';
+import {
+  type EventsQueuePayloadIncomingEvent,
+  getEventsGroupQueueShard,
+} from '@openpanel/queue';
 import { getRedisCache } from '@openpanel/redis';
 import {
   type IDecrementPayload,
@@ -112,6 +115,7 @@ interface TrackContext {
   identity?: IIdentifyPayload;
   deviceId: string;
   sessionId: string;
+  session?: EventsQueuePayloadIncomingEvent['payload']['session'];
   geo: GeoLocation;
 }
 
@@ -141,19 +145,21 @@ async function buildContext(
     validatedBody.payload.profileId = profileId;
   }
 
+  const overrideDeviceId =
+    validatedBody.type === 'track' &&
+    typeof validatedBody.payload?.properties?.__deviceId === 'string'
+      ? validatedBody.payload?.properties.__deviceId
+      : undefined;
+
   // Get geo location (needed for track and identify)
   const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
 
-  const { deviceId, sessionId } = await getDeviceId({
+  const deviceIdResult = await getDeviceId({
     projectId,
     ip,
     ua,
     salts,
-    overrideDeviceId:
+    overrideDeviceId,
-      validatedBody.type === 'track' &&
-      typeof validatedBody.payload?.properties?.__deviceId === 'string'
-        ? validatedBody.payload?.properties.__deviceId
-        : undefined,
   });
 
   return {
@@ -166,8 +172,9 @@ async function buildContext(
       isFromPast: timestamp.isTimestampFromThePast,
     },
     identity,
-    deviceId,
+    deviceId: deviceIdResult.deviceId,
-    sessionId,
+    sessionId: deviceIdResult.sessionId,
+    session: deviceIdResult.session,
     geo,
   };
 }
@@ -176,13 +183,14 @@ async function handleTrack(
   payload: ITrackPayload,
   context: TrackContext
 ): Promise<void> {
-  const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
+  const { projectId, deviceId, geo, headers, timestamp, sessionId, session } =
+    context;
 
   const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
   const groupId = uaInfo.isServer
     ? payload.profileId
       ? `${projectId}:${payload.profileId}`
-      : `${projectId}:${generateId()}`
+      : undefined
     : deviceId;
   const jobId = [
     slug(payload.name),
@@ -203,7 +211,7 @@ async function handleTrack(
   }
 
   promises.push(
-    getEventsGroupQueueShard(groupId).add({
+    getEventsGroupQueueShard(groupId || generateId()).add({
       orderMs: timestamp.value,
       data: {
         projectId,
@@ -217,6 +225,7 @@ async function handleTrack(
         geo,
         deviceId,
         sessionId,
+        session,
       },
       groupId,
       jobId,

apps/api/src/hooks/is-bot.hook.ts

@@ -1,20 +1,19 @@
-import { isBot } from '@/bots';
 import { createBotEvent } from '@openpanel/db';
 import type {
   DeprecatedPostEventPayload,
   ITrackHandlerPayload,
 } from '@openpanel/validation';
-
 import type { FastifyReply, FastifyRequest } from 'fastify';
+import { isBot } from '@/bots';
 
 export async function isBotHook(
   req: FastifyRequest<{
     Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
   }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const bot = req.headers['user-agent']
-    ? isBot(req.headers['user-agent'])
+    ? await isBot(req.headers['user-agent'])
     : null;
 
   if (bot && req.client?.projectId) {
@@ -44,6 +43,6 @@ export async function isBotHook(
       }
     }
 
-    return reply.status(202).send();
+    return reply.status(202).send({ bot });
   }
 }

apps/api/src/routes/track.router.ts

@@ -1,6 +1,5 @@
-import { fetchDeviceId, handler } from '@/controllers/track.controller';
 import type { FastifyPluginCallback } from 'fastify';
-
+import { fetchDeviceId, handler } from '@/controllers/track.controller';
 import { clientHook } from '@/hooks/client.hook';
 import { duplicateHook } from '@/hooks/duplicate.hook';
 import { isBotHook } from '@/hooks/is-bot.hook';
@@ -13,7 +12,7 @@ const trackRouter: FastifyPluginCallback = async (fastify) => {
   fastify.route({
     method: 'POST',
     url: '/',
-    handler: handler,
+    handler,
   });
 
   fastify.route({

apps/api/src/utils/ids.ts

@@ -1,7 +1,12 @@
 import crypto from 'node:crypto';
 import { generateDeviceId } from '@openpanel/common/server';
 import { getSafeJson } from '@openpanel/json';
+import type {
+  EventsQueuePayloadCreateSessionEnd,
+  EventsQueuePayloadIncomingEvent,
+} from '@openpanel/queue';
 import { getRedisCache } from '@openpanel/redis';
+import { pick } from 'ramda';
 
 export async function getDeviceId({
   projectId,
@@ -37,14 +42,20 @@ export async function getDeviceId({
     ua,
   });
 
-  return await getDeviceIdFromSession({
+  return await getInfoFromSession({
     projectId,
     currentDeviceId,
     previousDeviceId,
   });
 }
 
-async function getDeviceIdFromSession({
+interface DeviceIdResult {
+  deviceId: string;
+  sessionId: string;
+  session?: EventsQueuePayloadIncomingEvent['payload']['session'];
+}
+
+async function getInfoFromSession({
   projectId,
   currentDeviceId,
   previousDeviceId,
@@ -52,7 +63,7 @@ async function getDeviceIdFromSession({
   projectId: string;
   currentDeviceId: string;
   previousDeviceId: string;
-}) {
+}): Promise<DeviceIdResult> {
   try {
     const multi = getRedisCache().multi();
     multi.hget(
@@ -65,21 +76,33 @@ async function getDeviceIdFromSession({
     );
     const res = await multi.exec();
     if (res?.[0]?.[1]) {
-      const data = getSafeJson<{ payload: { sessionId: string } }>(
+      const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(
         (res?.[0]?.[1] as string) ?? ''
       );
       if (data) {
-        const sessionId = data.payload.sessionId;
+        return {
-        return { deviceId: currentDeviceId, sessionId };
+          deviceId: currentDeviceId,
+          sessionId: data.payload.sessionId,
+          session: pick(
+            ['referrer', 'referrerName', 'referrerType'],
+            data.payload
+          ),
+        };
       }
     }
     if (res?.[1]?.[1]) {
-      const data = getSafeJson<{ payload: { sessionId: string } }>(
+      const data = getSafeJson<EventsQueuePayloadCreateSessionEnd>(
         (res?.[1]?.[1] as string) ?? ''
       );
       if (data) {
-        const sessionId = data.payload.sessionId;
+        return {
-        return { deviceId: previousDeviceId, sessionId };
+          deviceId: previousDeviceId,
+          sessionId: data.payload.sessionId,
+          session: pick(
+            ['referrer', 'referrerName', 'referrerType'],
+            data.payload
+          ),
+        };
       }
     }
   } catch (error) {

apps/start/src/components/events/event-list-item.tsx

@@ -1,25 +1,20 @@
+import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
+import { Link } from '@tanstack/react-router';
+import { SerieIcon } from '../report-chart/common/serie-icon';
+import { EventIcon } from './event-icon';
 import { Tooltiper } from '@/components/ui/tooltip';
 import { useAppParams } from '@/hooks/use-app-params';
-import { useNumber } from '@/hooks/use-numer-formatter';
 import { pushModal } from '@/modals';
 import { cn } from '@/utils/cn';
 import { getProfileName } from '@/utils/getters';
-import { Link } from '@tanstack/react-router';
-
-import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
-
-import { SerieIcon } from '../report-chart/common/serie-icon';
-import { EventIcon } from './event-icon';
 
 type EventListItemProps = IServiceEventMinimal | IServiceEvent;
 
 export function EventListItem(props: EventListItemProps) {
   const { organizationId, projectId } = useAppParams();
-  const { createdAt, name, path, duration, meta } = props;
+  const { createdAt, name, path, meta } = props;
   const profile = 'profile' in props ? props.profile : null;
 
-  const number = useNumber();
-
   const renderName = () => {
     if (name === 'screen_view') {
       if (path.includes('/')) {
@@ -32,83 +27,65 @@ export function EventListItem(props: EventListItemProps) {
     return name.replace(/_/g, ' ');
   };
 
-  const renderDuration = () => {
-    if (name === 'screen_view') {
-      return (
-        <span className="text-muted-foreground">
-          {number.shortWithUnit(duration / 1000, 'min')}
-        </span>
-      );
-    }
-
-    return null;
-  };
-
   const isMinimal = 'minimal' in props;
 
   return (
-    <>
+    <button
-      <button
+      className={cn(
-        type="button"
+        'card flex w-full items-center justify-between rounded-lg p-4 transition-colors hover:bg-light-background',
-        onClick={() => {
+        meta?.conversion &&
-          if (!isMinimal) {
+          `bg-${meta.color}-50 dark:bg-${meta.color}-900 hover:bg-${meta.color}-100 dark:hover:bg-${meta.color}-700`
-            pushModal('EventDetails', {
+      )}
-              id: props.id,
+      onClick={() => {
-              projectId,
+        if (!isMinimal) {
-              createdAt,
+          pushModal('EventDetails', {
-            });
+            id: props.id,
-          }
+            projectId,
-        }}
+            createdAt,
-        className={cn(
+          });
-          'card hover:bg-light-background flex w-full items-center justify-between rounded-lg p-4 transition-colors',
+        }
-          meta?.conversion &&
+      }}
-            `bg-${meta.color}-50 dark:bg-${meta.color}-900 hover:bg-${meta.color}-100 dark:hover:bg-${meta.color}-700`,
+      type="button"
-        )}
+    >
-      >
+      <div>
-        <div>
+        <div className="flex items-center gap-4 text-left">
-          <div className="flex items-center gap-4 text-left ">
+          <EventIcon meta={meta} name={name} size="sm" />
-            <EventIcon size="sm" name={name} meta={meta} />
+          <span className="font-medium">{renderName()}</span>
-            <span>
+        </div>
-              <span className="font-medium">{renderName()}</span>
+        <div className="pl-10">
-              {'  '}
+          <div className="flex origin-left scale-75 gap-1">
-              {renderDuration()}
+            <SerieIcon name={props.country} />
-            </span>
+            <SerieIcon name={props.os} />
-          </div>
+            <SerieIcon name={props.browser} />
-          <div className="pl-10">
-            <div className="flex origin-left scale-75 gap-1">
-              <SerieIcon name={props.country} />
-              <SerieIcon name={props.os} />
-              <SerieIcon name={props.browser} />
-            </div>
           </div>
         </div>
-        <div className="flex gap-4">
+      </div>
-          {profile && (
+      <div className="flex gap-4">
-            <Tooltiper asChild content={getProfileName(profile)}>
+        {profile && (
-              <Link
+          <Tooltiper asChild content={getProfileName(profile)}>
-                onClick={(e) => {
+            <Link
-                  e.stopPropagation();
+              className="max-w-[80px] overflow-hidden text-ellipsis whitespace-nowrap text-muted-foreground hover:underline"
-                }}
+              onClick={(e) => {
-                to={'/$organizationId/$projectId/profiles/$profileId'}
+                e.stopPropagation();
-                params={{
+              }}
-                  organizationId,
+              params={{
-                  projectId,
+                organizationId,
-                  profileId: profile.id,
+                projectId,
-                }}
+                profileId: profile.id,
-                className="max-w-[80px] overflow-hidden text-ellipsis whitespace-nowrap  text-muted-foreground hover:underline"
+              }}
-              >
+              to={'/$organizationId/$projectId/profiles/$profileId'}
-                {getProfileName(profile)}
+            >
-              </Link>
+              {getProfileName(profile)}
-            </Tooltiper>
+            </Link>
-          )}
-
-          <Tooltiper asChild content={createdAt.toLocaleString()}>
-            <div className=" text-muted-foreground">
-              {createdAt.toLocaleTimeString()}
-            </div>
           </Tooltiper>
-        </div>
+        )}
-      </button>
+
-    </>
+        <Tooltiper asChild content={createdAt.toLocaleString()}>
+          <div className="text-muted-foreground">
+            {createdAt.toLocaleTimeString()}
+          </div>
+        </Tooltiper>
+      </div>
+    </button>
   );
 }

apps/start/src/components/events/event-listener.tsx

@@ -1,3 +1,4 @@
+import { AnimatedNumber } from '../animated-number';
 import {
   Tooltip,
   TooltipContent,
@@ -8,71 +9,53 @@ import { useDebounceState } from '@/hooks/use-debounce-state';
 import useWS from '@/hooks/use-ws';
 import { cn } from '@/utils/cn';
 
-import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
-import { useParams } from '@tanstack/react-router';
-import { AnimatedNumber } from '../animated-number';
-
 export default function EventListener({
   onRefresh,
 }: {
   onRefresh: () => void;
 }) {
-  const params = useParams({
-    strict: false,
-  });
   const { projectId } = useAppParams();
   const counter = useDebounceState(0, 1000);
-  useWS<IServiceEventMinimal | IServiceEvent>(
+  useWS<{ count: number }>(
     `/live/events/${projectId}`,
-    (event) => {
+    ({ count }) => {
-      if (event) {
+      counter.set((prev) => prev + count);
-        const isProfilePage = !!params?.profileId;
-        if (isProfilePage) {
-          const profile = 'profile' in event ? event.profile : null;
-          if (profile?.id === params?.profileId) {
-            counter.set((prev) => prev + 1);
-          }
-          return;
-        }
-
-        counter.set((prev) => prev + 1);
-      }
     },
     {
       debounce: {
         delay: 1000,
         maxWait: 5000,
       },
-    },
+    }
   );
 
   return (
     <Tooltip>
       <TooltipTrigger asChild>
         <button
-          type="button"
+          className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
           onClick={() => {
             counter.set(0);
             onRefresh();
           }}
-          className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
+          type="button"
         >
           <div className="relative">
             <div
               className={cn(
-                'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
+                'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all'
               )}
             />
             <div
               className={cn(
-                'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
+                'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all'
               )}
             />
           </div>
           {counter.debounced === 0 ? (
             'Listening'
           ) : (
-            <AnimatedNumber value={counter.debounced} suffix=" new events" />
+            <AnimatedNumber suffix=" new events" value={counter.debounced} />
           )}
         </button>
       </TooltipTrigger>

apps/start/src/components/events/table/columns.tsx

@@ -1,15 +1,14 @@
+import type { IServiceEvent } from '@openpanel/db';
+import type { ColumnDef } from '@tanstack/react-table';
+import { ColumnCreatedAt } from '@/components/column-created-at';
 import { EventIcon } from '@/components/events/event-icon';
 import { ProjectLink } from '@/components/links';
+import { ProfileAvatar } from '@/components/profiles/profile-avatar';
 import { SerieIcon } from '@/components/report-chart/common/serie-icon';
+import { KeyValueGrid } from '@/components/ui/key-value-grid';
 import { useNumber } from '@/hooks/use-numer-formatter';
 import { pushModal } from '@/modals';
 import { getProfileName } from '@/utils/getters';
-import type { ColumnDef } from '@tanstack/react-table';
-
-import { ColumnCreatedAt } from '@/components/column-created-at';
-import { ProfileAvatar } from '@/components/profiles/profile-avatar';
-import { KeyValueGrid } from '@/components/ui/key-value-grid';
-import type { IServiceEvent } from '@openpanel/db';
 
 export function useColumns() {
   const number = useNumber();
@@ -28,17 +27,24 @@ export function useColumns() {
       accessorKey: 'name',
       header: 'Name',
       cell({ row }) {
-        const { name, path, duration, properties, revenue } = row.original;
+        const { name, path, revenue } = row.original;
+        const fullTitle =
+          name === 'screen_view'
+            ? path
+            : name === 'revenue' && revenue
+              ? `${name} (${number.currency(revenue / 100)})`
+              : name.replace(/_/g, ' ');
+
         const renderName = () => {
           if (name === 'screen_view') {
             if (path.includes('/')) {
-              return <span className="max-w-md truncate">{path}</span>;
+              return path;
             }
 
             return (
               <>
                 <span className="text-muted-foreground">Screen: </span>
-                <span className="max-w-md truncate">{path}</span>
+                {path}
               </>
             );
           }
@@ -50,38 +56,27 @@ export function useColumns() {
           return name.replace(/_/g, ' ');
         };
 
-        const renderDuration = () => {
-          if (name === 'screen_view') {
-            return (
-              <span className="text-muted-foreground">
-                {number.shortWithUnit(duration / 1000, 'min')}
-              </span>
-            );
-          }
-
-          return null;
-        };
-
         return (
-          <div className="flex items-center gap-2">
+          <div className="flex min-w-0 items-center gap-2">
             <button
-              type="button"
+              className="shrink-0 transition-transform hover:scale-105"
-              className="transition-transform hover:scale-105"
               onClick={() => {
                 pushModal('EditEvent', {
                   id: row.original.id,
                 });
               }}
+              type="button"
             >
               <EventIcon
-                size="sm"
-                name={row.original.name}
                 meta={row.original.meta}
+                name={row.original.name}
+                size="sm"
               />
             </button>
-            <span className="flex gap-2">
+            <span className="flex min-w-0 flex-1 gap-2">
               <button
-                type="button"
+                className="min-w-0 max-w-full truncate text-left font-medium hover:underline"
+                title={fullTitle}
                 onClick={() => {
                   pushModal('EventDetails', {
                     id: row.original.id,
@@ -89,11 +84,10 @@ export function useColumns() {
                     projectId: row.original.projectId,
                   });
                 }}
-                className="font-medium hover:underline"
+                type="button"
               >
-                {renderName()}
+                <span className="block truncate">{renderName()}</span>
               </button>
-              {renderDuration()}
             </span>
           </div>
         );
@@ -107,8 +101,8 @@ export function useColumns() {
         if (profile) {
           return (
             <ProjectLink
+              className="group row items-center gap-2 whitespace-nowrap font-medium hover:underline"
               href={`/profiles/${encodeURIComponent(profile.id)}`}
-              className="group whitespace-nowrap font-medium hover:underline row items-center gap-2"
             >
               <ProfileAvatar size="sm" {...profile} />
               {getProfileName(profile)}
@@ -119,8 +113,8 @@ export function useColumns() {
         if (profileId && profileId !== deviceId) {
           return (
             <ProjectLink
-              href={`/profiles/${encodeURIComponent(profileId)}`}
               className="whitespace-nowrap font-medium hover:underline"
+              href={`/profiles/${encodeURIComponent(profileId)}`}
             >
               Unknown
             </ProjectLink>
@@ -130,8 +124,8 @@ export function useColumns() {
         if (deviceId) {
           return (
             <ProjectLink
-              href={`/profiles/${encodeURIComponent(deviceId)}`}
               className="whitespace-nowrap font-medium hover:underline"
+              href={`/profiles/${encodeURIComponent(deviceId)}`}
             >
               Anonymous
             </ProjectLink>
@@ -152,10 +146,10 @@ export function useColumns() {
         const { sessionId } = row.original;
         return (
           <ProjectLink
-            href={`/sessions/${encodeURIComponent(sessionId)}`}
             className="whitespace-nowrap font-medium hover:underline"
+            href={`/sessions/${encodeURIComponent(sessionId)}`}
           >
-            {sessionId.slice(0,6)}
+            {sessionId.slice(0, 6)}
           </ProjectLink>
         );
       },
@@ -175,7 +169,7 @@ export function useColumns() {
       cell({ row }) {
         const { country, city } = row.original;
         return (
-          <div className="row items-center gap-2 min-w-0">
+          <div className="row min-w-0 items-center gap-2">
             <SerieIcon name={country} />
             <span className="truncate">{city}</span>
           </div>
@@ -189,7 +183,7 @@ export function useColumns() {
       cell({ row }) {
         const { os } = row.original;
         return (
-          <div className="row items-center gap-2 min-w-0">
+          <div className="row min-w-0 items-center gap-2">
             <SerieIcon name={os} />
             <span className="truncate">{os}</span>
           </div>
@@ -203,7 +197,7 @@ export function useColumns() {
       cell({ row }) {
         const { browser } = row.original;
         return (
-          <div className="row items-center gap-2 min-w-0">
+          <div className="row min-w-0 items-center gap-2">
             <SerieIcon name={browser} />
             <span className="truncate">{browser}</span>
           </div>
@@ -221,14 +215,14 @@ export function useColumns() {
         const { properties } = row.original;
         const filteredProperties = Object.fromEntries(
           Object.entries(properties || {}).filter(
-            ([key]) => !key.startsWith('__'),
+            ([key]) => !key.startsWith('__')
-          ),
+          )
         );
         const items = Object.entries(filteredProperties);
         const limit = 2;
         const data = items.slice(0, limit).map(([key, value]) => ({
           name: key,
-          value: value,
+          value,
         }));
         if (items.length > limit) {
           data.push({

apps/start/src/components/events/table/index.tsx

@@ -35,6 +35,7 @@ type Props = {
     >,
     unknown
   >;
+  showEventListener?: boolean;
 };
 
 const LOADING_DATA = [{}, {}, {}, {}, {}, {}, {}, {}, {}] as IServiceEvent[];
@@ -215,7 +216,7 @@ const VirtualizedEventsTable = ({
   );
 };
 
-export const EventsTable = ({ query }: Props) => {
+export const EventsTable = ({ query, showEventListener = false }: Props) => {
   const { isLoading } = query;
   const columns = useColumns();
 
@@ -272,7 +273,7 @@ export const EventsTable = ({ query }: Props) => {
 
   return (
     <>
-      <EventsTableToolbar query={query} table={table} />
+      <EventsTableToolbar query={query} table={table} showEventListener={showEventListener} />
       <VirtualizedEventsTable table={table} data={data} isLoading={isLoading} />
       <div className="w-full h-10 center-center pt-4" ref={inViewportRef}>
         <div
@@ -291,9 +292,11 @@ export const EventsTable = ({ query }: Props) => {
 function EventsTableToolbar({
   query,
   table,
+  showEventListener,
 }: {
   query: Props['query'];
   table: Table<IServiceEvent>;
+  showEventListener: boolean;
 }) {
   const { projectId } = useAppParams();
   const [startDate, setStartDate] = useQueryState(
@@ -305,7 +308,7 @@ function EventsTableToolbar({
   return (
     <DataTableToolbarContainer>
       <div className="flex flex-1 flex-wrap items-center gap-2">
-        <EventListener onRefresh={() => query.refetch()} />
+        {showEventListener && <EventListener onRefresh={() => query.refetch()} />}
         <Button
           variant="outline"
           size="sm"

apps/start/src/components/onboarding/onboarding-verify-listener.tsx

@@ -1,31 +1,13 @@
-import type {
+import type { IServiceEvent } from '@openpanel/db';
-  IServiceClient,
-  IServiceEvent,
-  IServiceProject,
-} from '@openpanel/db';
 import { CheckCircle2Icon, CheckIcon, Loader2 } from 'lucide-react';
-import { useState } from 'react';
-import useWS from '@/hooks/use-ws';
 import { cn } from '@/utils/cn';
 import { timeAgo } from '@/utils/date';
 
 interface Props {
-  project: IServiceProject;
-  client: IServiceClient | null;
   events: IServiceEvent[];
-  onVerified: (verified: boolean) => void;
 }
 
-const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
+const VerifyListener = ({ events }: Props) => {
-  const [events, setEvents] = useState<IServiceEvent[]>(_events ?? []);
-  useWS<IServiceEvent>(
-    `/live/events/${client?.projectId}?type=received`,
-    (data) => {
-      setEvents((prev) => [...prev, data]);
-      onVerified(true);
-    }
-  );
-
   const isConnected = events.length > 0;
 
   const renderIcon = () => {
@@ -49,16 +31,18 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
       <div
         className={cn(
           'flex gap-6 rounded-xl p-4 md:p-6',
-          isConnected ? 'bg-emerald-100 dark:bg-emerald-700' : 'bg-blue-500/10'
+          isConnected
+            ? 'bg-emerald-100 dark:bg-emerald-700/10'
+            : 'bg-blue-500/10'
         )}
       >
         {renderIcon()}
         <div className="flex-1">
           <div className="font-semibold text-foreground/90 text-lg leading-normal">
-            {isConnected ? 'Success' : 'Waiting for events'}
+            {isConnected ? 'Successfully connected' : 'Waiting for events'}
           </div>
           {isConnected ? (
-            <div className="flex flex-col-reverse">
+            <div className="mt-2 flex flex-col-reverse gap-1">
               {events.length > 5 && (
                 <div className="flex items-center gap-2">
                   <CheckIcon size={14} />{' '}
@@ -69,7 +53,7 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
                 <div className="flex items-center gap-2" key={event.id}>
                   <CheckIcon size={14} />{' '}
                   <span className="font-medium">{event.name}</span>{' '}
-                  <span className="ml-auto text-emerald-800">
+                  <span className="ml-auto text-foreground/50 text-sm">
                     {timeAgo(event.createdAt, 'round')}
                   </span>
                 </div>

apps/start/src/components/realtime/realtime-active-sessions.tsx

@@ -1,11 +1,9 @@
-import { useTRPC } from '@/integrations/trpc/react';
 import { useQuery } from '@tanstack/react-query';
 import { AnimatePresence, motion } from 'framer-motion';
-import { useEffect, useState } from 'react';
+import { ProjectLink } from '../links';
-
+import { SerieIcon } from '../report-chart/common/serie-icon';
-import useWS from '@/hooks/use-ws';
+import { useTRPC } from '@/integrations/trpc/react';
-import type { IServiceEvent } from '@openpanel/db';
+import { formatTimeAgoOrDateTime } from '@/utils/date';
-import { EventItem } from '../events/table/item';
 
 interface RealtimeActiveSessionsProps {
   projectId: string;
@@ -17,64 +15,52 @@ export function RealtimeActiveSessions({
   limit = 10,
 }: RealtimeActiveSessionsProps) {
   const trpc = useTRPC();
-  const activeSessionsQuery = useQuery(
+  const { data: sessions = [] } = useQuery(
-    trpc.realtime.activeSessions.queryOptions({
+    trpc.realtime.activeSessions.queryOptions(
-      projectId,
+      { projectId },
-    }),
+      { refetchInterval: 5000 }
+    )
   );
 
-  const [state, setState] = useState<IServiceEvent[]>([]);
-
-  // Update state when initial data loads
-  useEffect(() => {
-    if (activeSessionsQuery.data && state.length === 0) {
-      setState(activeSessionsQuery.data);
-    }
-  }, [activeSessionsQuery.data, state]);
-
-  // Set up WebSocket connection for real-time updates
-  useWS<IServiceEvent>(
-    `/live/events/${projectId}`,
-    (session) => {
-      setState((prev) => {
-        // Add new session and remove duplicates, keeping most recent
-        const filtered = prev.filter((s) => s.id !== session.id);
-        return [session, ...filtered].slice(0, limit);
-      });
-    },
-    {
-      debounce: {
-        delay: 1000,
-        maxWait: 5000,
-      },
-    },
-  );
-
-  const sessions = state.length > 0 ? state : (activeSessionsQuery.data ?? []);
-
   return (
-    <div className="col h-full max-md:hidden">
+    <div className="col card h-full max-md:hidden">
-      <div className="hide-scrollbar h-full overflow-y-auto pb-10">
+      <div className="hide-scrollbar h-full overflow-y-auto">
-        <AnimatePresence mode="popLayout" initial={false}>
+        <AnimatePresence initial={false} mode="popLayout">
-          <div className="col gap-4">
+          <div className="col divide-y">
-            {sessions.map((session) => (
+            {sessions.slice(0, limit).map((session) => (
               <motion.div
-                key={session.id}
-                layout
-                // initial={{ opacity: 0, x: -200, scale: 0.8 }}
                 animate={{ opacity: 1, x: 0, scale: 1 }}
                 exit={{ opacity: 0, x: 200, scale: 0.8 }}
+                key={session.id}
+                layout
                 transition={{ duration: 0.4, type: 'spring', stiffness: 300 }}
               >
-                <EventItem
+                <ProjectLink
-                  event={session}
+                  className="relative block p-4 py-3 pr-14"
-                  viewOptions={{
+                  href={`/sessions/${session.sessionId}`}
-                    properties: false,
+                >
-                    origin: false,
+                  <div className="col flex-1 gap-1">
-                    queryString: false,
+                    {session.name === 'screen_view' && (
-                  }}
+                      <span className="text-muted-foreground text-xs leading-normal/80">
-                  className="w-full"
+                        {session.origin}
-                />
+                      </span>
+                    )}
+                    <span className="font-medium text-sm leading-normal">
+                      {session.name === 'screen_view'
+                        ? session.path
+                        : session.name}
+                    </span>
+                    <span className="text-muted-foreground text-xs">
+                      {formatTimeAgoOrDateTime(session.createdAt)}
+                    </span>
+                  </div>
+                  <div className="row absolute top-1/2 right-4 origin-right -translate-y-1/2 scale-50 gap-2">
+                    <SerieIcon name={session.referrerName} />
+                    <SerieIcon name={session.os} />
+                    <SerieIcon name={session.browser} />
+                    <SerieIcon name={session.device} />
+                  </div>
+                </ProjectLink>
               </motion.div>
             ))}
           </div>

apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx

@@ -42,5 +42,5 @@ function Component() {
     ),
   );
 
-  return <EventsTable query={query} />;
+  return <EventsTable query={query} showEventListener />;
 }

apps/start/src/routes/_app.$organizationId.$projectId.realtime.tsx

@@ -1,3 +1,5 @@
+import { keepPreviousData, useQuery } from '@tanstack/react-query';
+import { createFileRoute } from '@tanstack/react-router';
 import { Fullscreen, FullscreenClose } from '@/components/fullscreen-toggle';
 import RealtimeMap from '@/components/realtime/map';
 import { RealtimeActiveSessions } from '@/components/realtime/realtime-active-sessions';
@@ -7,12 +9,10 @@ import { RealtimePaths } from '@/components/realtime/realtime-paths';
 import { RealtimeReferrals } from '@/components/realtime/realtime-referrals';
 import RealtimeReloader from '@/components/realtime/realtime-reloader';
 import { useTRPC } from '@/integrations/trpc/react';
-import { PAGE_TITLES, createProjectTitle } from '@/utils/title';
+import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
-import { keepPreviousData, useQuery } from '@tanstack/react-query';
-import { createFileRoute } from '@tanstack/react-router';
 
 export const Route = createFileRoute(
-  '/_app/$organizationId/$projectId/realtime',
+  '/_app/$organizationId/$projectId/realtime'
 )({
   component: Component,
   head: () => {
@@ -36,8 +36,8 @@ function Component() {
       },
       {
         placeholderData: keepPreviousData,
-      },
+      }
-    ),
+    )
   );
 
   return (
@@ -47,7 +47,7 @@ function Component() {
         <RealtimeReloader projectId={projectId} />
 
         <div className="row relative">
-          <div className="overflow-hidden aspect-[4/2] w-full">
+          <div className="aspect-[4/2] w-full overflow-hidden">
             <RealtimeMap
               markers={coordinatesQuery.data ?? []}
               sidebarConfig={{
@@ -56,18 +56,17 @@ function Component() {
               }}
             />
           </div>
-          <div className="absolute top-8 left-8 bottom-0 col gap-4">
+          <div className="col absolute top-8 bottom-4 left-8 gap-4">
-            <div className="card p-4 w-72 bg-background/90">
+            <div className="card w-72 bg-background/90 p-4">
               <RealtimeLiveHistogram projectId={projectId} />
             </div>
-            <div className="w-72 flex-1 min-h-0 relative">
+            <div className="relative min-h-0 w-72 flex-1">
               <RealtimeActiveSessions projectId={projectId} />
-              <div className="absolute bottom-0 left-0 right-0 h-10 bg-gradient-to-t from-def-100 to-transparent" />
             </div>
           </div>
         </div>
 
-        <div className="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4 p-4 pt-4 md:p-8 md:pt-0">
+        <div className="grid grid-cols-1 gap-4 p-4 pt-4 md:grid-cols-2 md:p-8 md:pt-0 xl:grid-cols-3">
           <div>
             <RealtimeGeo projectId={projectId} />
           </div>

apps/start/src/routes/_steps.onboarding.$projectId.verify.tsx

@@ -1,7 +1,6 @@
 import { useQuery } from '@tanstack/react-query';
 import { createFileRoute, Link, redirect } from '@tanstack/react-router';
 import { BoxSelectIcon } from 'lucide-react';
-import { useEffect, useState } from 'react';
 import { ButtonContainer } from '@/components/button-container';
 import { FullPageEmptyState } from '@/components/full-page-empty-state';
 import FullPageLoadingState from '@/components/full-page-loading-state';
@@ -33,22 +32,21 @@ export const Route = createFileRoute('/_steps/onboarding/$projectId/verify')({
 });
 
 function Component() {
-  const [isVerified, setIsVerified] = useState(false);
   const { projectId } = Route.useParams();
   const trpc = useTRPC();
-  const { data: events, refetch } = useQuery(
+  const { data: events } = useQuery(
-    trpc.event.events.queryOptions({ projectId })
+    trpc.event.events.queryOptions(
+      { projectId },
+      {
+        refetchInterval: 2500,
+      }
+    )
   );
+  const isVerified = events?.data && events.data.length > 0;
   const { data: project } = useQuery(
     trpc.project.getProjectWithClients.queryOptions({ projectId })
   );
 
-  useEffect(() => {
-    if (events && events.data.length > 0) {
-      setIsVerified(true);
-    }
-  }, [events]);
-
   if (!project) {
     return (
       <FullPageEmptyState icon={BoxSelectIcon} title="Project not found" />
@@ -64,15 +62,7 @@ function Component() {
     <div className="flex min-h-0 flex-1 flex-col">
       <div className="scrollbar-thin flex-1 overflow-y-auto">
         <div className="col gap-8 p-4">
-          <VerifyListener
+          <VerifyListener events={events?.data ?? []} />
-            client={client}
-            events={events?.data ?? []}
-            onVerified={() => {
-              refetch();
-              setIsVerified(true);
-            }}
-            project={project}
-          />
 
           <VerifyFaq project={project} />
         </div>

apps/worker/package.json

@@ -16,11 +16,11 @@
     "@openpanel/common": "workspace:*",
     "@openpanel/db": "workspace:*",
     "@openpanel/email": "workspace:*",
+    "@openpanel/importer": "workspace:*",
     "@openpanel/integrations": "workspace:^",
     "@openpanel/js-runtime": "workspace:*",
     "@openpanel/json": "workspace:*",
     "@openpanel/logger": "workspace:*",
-    "@openpanel/importer": "workspace:*",
     "@openpanel/payments": "workspace:*",
     "@openpanel/queue": "workspace:*",
     "@openpanel/redis": "workspace:*",

apps/worker/src/boot-workers.ts

@@ -1,10 +1,9 @@
-import type { Queue, WorkerOptions } from 'bullmq';
+import { performance } from 'node:perf_hooks';
-import { Worker } from 'bullmq';
+import { setTimeout as sleep } from 'node:timers/promises';
-
 import {
+  cronQueue,
   EVENTS_GROUP_QUEUES_SHARDS,
   type EventsQueuePayloadIncomingEvent,
-  cronQueue,
   eventsGroupQueues,
   gscQueue,
   importQueue,
@@ -15,14 +14,12 @@ import {
   sessionsQueue,
 } from '@openpanel/queue';
 import { getRedisQueue } from '@openpanel/redis';
-
+import type { Queue, WorkerOptions } from 'bullmq';
-import { performance } from 'node:perf_hooks';
+import { Worker } from 'bullmq';
-import { setTimeout as sleep } from 'node:timers/promises';
 import { Worker as GroupWorker } from 'groupmq';
-
 import { cronJob } from './jobs/cron';
-import { gscJob } from './jobs/gsc';
 import { incomingEvent } from './jobs/events.incoming-event';
+import { gscJob } from './jobs/gsc';
 import { importJob } from './jobs/import';
 import { insightsProjectJob } from './jobs/insights';
 import { miscJob } from './jobs/misc';
@@ -95,7 +92,7 @@ function getConcurrencyFor(queueName: string, defaultValue = 1): number {
   return defaultValue;
 }
 
-export async function bootWorkers() {
+export function bootWorkers() {
   const enabledQueues = getEnabledQueues();
 
   const workers: (Worker | GroupWorker<any>)[] = [];
@@ -119,12 +116,14 @@ export async function bootWorkers() {
 
   for (const index of eventQueuesToStart) {
     const queue = eventsGroupQueues[index];
-    if (!queue) continue;
+    if (!queue) {
+      continue;
+    }
 
     const queueName = `events_${index}`;
     const concurrency = getConcurrencyFor(
       queueName,
-      Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10),
+      Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10)
     );
 
     const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
@@ -132,7 +131,7 @@ export async function bootWorkers() {
       concurrency,
       logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,
       blockingTimeoutSec: Number.parseFloat(
-        process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',
+        process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1'
       ),
       handler: async (job) => {
         return await incomingEvent(job.data);
@@ -172,7 +171,7 @@ export async function bootWorkers() {
     const notificationWorker = new Worker(
       notificationQueue.name,
       notificationJob,
-      { ...workerOptions, concurrency },
+      { ...workerOptions, concurrency }
     );
     workers.push(notificationWorker);
     logger.info('Started worker for notification', { concurrency });
@@ -224,7 +223,7 @@ export async function bootWorkers() {
 
   if (workers.length === 0) {
     logger.warn(
-      'No workers started. Check ENABLED_QUEUES environment variable.',
+      'No workers started. Check ENABLED_QUEUES environment variable.'
     );
   }
 
@@ -254,7 +253,7 @@ export async function bootWorkers() {
           const elapsed = job.finishedOn - job.processedOn;
           eventsGroupJobDuration.observe(
             { name: worker.name, status: 'failed' },
-            elapsed,
+            elapsed
           );
         }
         logger.error('job failed', {
@@ -267,23 +266,6 @@ export async function bootWorkers() {
       }
     });
 
-    (worker as Worker).on('completed', (job) => {
-      if (job) {
-        if (job.processedOn && job.finishedOn) {
-          const elapsed = job.finishedOn - job.processedOn;
-          logger.info('job completed', {
-            jobId: job.id,
-            worker: worker.name,
-            elapsed,
-          });
-          eventsGroupJobDuration.observe(
-            { name: worker.name, status: 'success' },
-            elapsed,
-          );
-        }
-      }
-    });
-
     (worker as Worker).on('ioredis:close', () => {
       logger.error('worker closed due to ioredis:close', {
         worker: worker.name,
@@ -293,7 +275,7 @@ export async function bootWorkers() {
 
   async function exitHandler(
     eventName: string,
-    evtOrExitCodeOrError: number | string | Error,
+    evtOrExitCodeOrError: number | string | Error
   ) {
     // Log the actual error details for unhandled rejections/exceptions
     if (evtOrExitCodeOrError instanceof Error) {
@@ -339,7 +321,7 @@ export async function bootWorkers() {
       process.on(evt, (code) => {
         exitHandler(evt, code);
       });
-    },
+    }
   );
 
   return workers;

apps/worker/src/jobs/cron.salt.ts

@@ -33,7 +33,7 @@ async function generateNewSalt() {
     return created;
   });
 
-  getSalts.clear();
+  await getSalts.clear();
 
   return newSalt;
 }

apps/worker/src/jobs/events.incoming-event.ts

@@ -1,9 +1,5 @@
 import { getTime, isSameDomain, parsePath } from '@openpanel/common';
-import {
+import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';
-  getReferrerWithQuery,
-  parseReferrer,
-  parseUserAgent,
-} from '@openpanel/common/server';
 import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
 import {
   checkNotificationRulesForEvent,
@@ -14,10 +10,12 @@ import {
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
-import { getLock } from '@openpanel/redis';
 import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';
 import { logger as baseLogger } from '@/utils/logger';
-import { createSessionEndJob, getSessionEnd } from '@/utils/session-handler';
+import {
+  createSessionEndJob,
+  extendSessionEndJob,
+} from '@/utils/session-handler';
 
 const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];
 
@@ -93,7 +91,8 @@ export async function incomingEvent(
     projectId,
     deviceId,
     sessionId,
-    uaInfo: _uaInfo,
+    uaInfo,
+    session,
   } = jobPayload;
   const properties = body.properties ?? {};
   const reqId = headers['request-id'] ?? 'unknown';
@@ -121,16 +120,15 @@ export async function incomingEvent(
     ? null
     : parseReferrer(getProperty('__referrer'));
   const utmReferrer = getReferrerWithQuery(query);
-  const userAgent = headers['user-agent'];
   const sdkName = headers['openpanel-sdk-name'];
   const sdkVersion = headers['openpanel-sdk-version'];
-  // TODO: Remove both user-agent and parseUserAgent
-  const uaInfo = _uaInfo ?? parseUserAgent(userAgent, properties);
 
-  const baseEvent = {
+  const baseEvent: IServiceCreateEventPayload = {
     name: body.name,
     profileId,
     projectId,
+    deviceId,
+    sessionId,
     properties: omit(GLOBAL_PROPERTIES, {
       ...properties,
       __hash: hash,
@@ -149,7 +147,7 @@ export async function incomingEvent(
     origin,
     referrer: referrer?.url || '',
     referrerName: utmReferrer?.name || referrer?.name || referrer?.url,
-    referrerType: referrer?.type || utmReferrer?.type || '',
+    referrerType: utmReferrer?.type || referrer?.type || '',
     os: uaInfo.os,
     osVersion: uaInfo.osVersion,
     browser: uaInfo.browser,
@@ -161,16 +159,17 @@ export async function incomingEvent(
       body.name === 'revenue' && '__revenue' in properties
         ? parseRevenue(properties.__revenue)
         : undefined,
-  } as const;
+  };
 
   // if timestamp is from the past we dont want to create a new session
   if (uaInfo.isServer || isTimestampFromThePast) {
-    const session = profileId
+    const session =
-      ? await sessionBuffer.getExistingSession({
+      profileId && !isTimestampFromThePast
-          profileId,
+        ? await sessionBuffer.getExistingSession({
-          projectId,
+            profileId,
-        })
+            projectId,
-      : null;
+          })
+        : null;
 
     const payload = {
       ...baseEvent,
@@ -198,82 +197,48 @@ export async function incomingEvent(
     return createEventAndNotify(payload as IServiceEvent, logger, projectId);
   }
 
-  const sessionEnd = await getSessionEnd({
-    projectId,
-    deviceId,
-    profileId,
-  });
-  const activeSession = sessionEnd
-    ? await sessionBuffer.getExistingSession({
-        sessionId: sessionEnd.sessionId,
-      })
-    : null;
-
   const payload: IServiceCreateEventPayload = merge(baseEvent, {
-    deviceId: sessionEnd?.deviceId ?? deviceId,
+    referrer: session?.referrer ?? baseEvent.referrer,
-    sessionId: sessionEnd?.sessionId ?? sessionId,
+    referrerName: session?.referrerName ?? baseEvent.referrerName,
-    referrer: sessionEnd?.referrer ?? baseEvent.referrer,
+    referrerType: session?.referrerType ?? baseEvent.referrerType,
-    referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,
-    referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,
-    // if the path is not set, use the last screen view path
-    path: baseEvent.path || activeSession?.exit_path || '',
-    origin: baseEvent.origin || activeSession?.exit_origin || '',
   } as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;
 
-  // If the triggering event is filtered, do not create session_start or the event (issue #2)
   const isExcluded = await isEventExcludedByProjectFilter(payload, projectId);
   if (isExcluded) {
     logger.info(
       'Skipping session_start and event (excluded by project filter)',
+      { event: payload.name, projectId }
+    );
+    return null;
+  }
+
+  if (session) {
+    await extendSessionEndJob({
+      projectId,
+      deviceId,
+    }).catch((error) => {
+      logger.error('Error finding and extending session end job', { error });
+      throw error;
+    });
+  } else {
+    await createEventAndNotify(
       {
-        event: payload.name,
+        ...payload,
-        projectId,
+        name: 'session_start',
-      }
+        createdAt: new Date(getTime(payload.createdAt) - 100),
-    );
+      },
-    return null;
+      logger,
-  }
+      projectId
+    ).catch((error) => {
+      logger.error('Error creating session start event', { event: payload });
+      throw error;
+    });
 
-  if (!sessionEnd) {
-    const locked = await getLock(
-      `session_start:${projectId}:${sessionId}`,
-      '1',
-      1000
-    );
-    if (locked) {
-      logger.info('Creating session start event', { event: payload });
-      await createEventAndNotify(
-        {
-          ...payload,
-          name: 'session_start',
-          createdAt: new Date(getTime(payload.createdAt) - 100),
-        },
-        logger,
-        projectId
-      ).catch((error) => {
-        logger.error('Error creating session start event', { event: payload });
-        throw error;
-      });
-    } else {
-      logger.info('Session start already claimed by another worker', {
-        event: payload,
-      });
-    }
-  }
-
-  const event = await createEventAndNotify(payload, logger, projectId);
-
-  if (!event) {
-    // Skip creating session end when event was excluded
-    return null;
-  }
-
-  if (!sessionEnd) {
-    logger.info('Creating session end job', { event: payload });
     await createSessionEndJob({ payload }).catch((error) => {
       logger.error('Error creating session end job', { event: payload });
       throw error;
     });
   }
 
-  return event;
+  return createEventAndNotify(payload, logger, projectId);
 }

apps/worker/src/jobs/events.incoming-events.test.ts

@@ -186,6 +186,11 @@ describe('incomingEvent', () => {
       projectId,
       deviceId,
       sessionId: 'session-123',
+      session: {
+        referrer: '',
+        referrerName: '',
+        referrerType: '',
+      },
     };
 
     const changeDelay = vi.fn();

apps/worker/src/metrics.ts

@@ -1,5 +1,3 @@
-import client from 'prom-client';
-
 import {
   botBuffer,
   eventBuffer,
@@ -8,6 +6,7 @@ import {
   sessionBuffer,
 } from '@openpanel/db';
 import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
+import client from 'prom-client';
 
 const Registry = client.Registry;
 
@@ -20,7 +19,7 @@ export const eventsGroupJobDuration = new client.Histogram({
   name: 'job_duration_ms',
   help: 'Duration of job processing (in ms)',
   labelNames: ['name', 'status'],
-  buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
+  buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10_000, 30_000], // 10ms to 30s
 });
 
 register.registerMetric(eventsGroupJobDuration);
@@ -28,57 +27,61 @@ register.registerMetric(eventsGroupJobDuration);
 queues.forEach((queue) => {
   register.registerMetric(
     new client.Gauge({
-      name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
+      name: `${queue.name.replace(/[{}]/g, '')}_active_count`,
       help: 'Active count',
       async collect() {
         const metric = await queue.getActiveCount();
         this.set(metric);
       },
-    }),
+    })
   );
 
   register.registerMetric(
     new client.Gauge({
-      name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
+      name: `${queue.name.replace(/[{}]/g, '')}_delayed_count`,
       help: 'Delayed count',
       async collect() {
-        const metric = await queue.getDelayedCount();
+        if ('getDelayedCount' in queue) {
-        this.set(metric);
+          const metric = await queue.getDelayedCount();
+          this.set(metric);
+        } else {
+          this.set(0);
+        }
       },
-    }),
+    })
   );
 
   register.registerMetric(
     new client.Gauge({
-      name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
+      name: `${queue.name.replace(/[{}]/g, '')}_failed_count`,
       help: 'Failed count',
       async collect() {
         const metric = await queue.getFailedCount();
         this.set(metric);
       },
-    }),
+    })
   );
 
   register.registerMetric(
     new client.Gauge({
-      name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
+      name: `${queue.name.replace(/[{}]/g, '')}_completed_count`,
       help: 'Completed count',
       async collect() {
         const metric = await queue.getCompletedCount();
         this.set(metric);
       },
-    }),
+    })
   );
 
   register.registerMetric(
     new client.Gauge({
-      name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
+      name: `${queue.name.replace(/[{}]/g, '')}_waiting_count`,
       help: 'Waiting count',
       async collect() {
         const metric = await queue.getWaitingCount();
         this.set(metric);
       },
-    }),
+    })
   );
 });
 
@@ -90,7 +93,7 @@ register.registerMetric(
       const metric = await eventBuffer.getBufferSize();
       this.set(metric);
     },
-  }),
+  })
 );
 
 register.registerMetric(
@@ -101,7 +104,7 @@ register.registerMetric(
       const metric = await profileBuffer.getBufferSize();
       this.set(metric);
     },
-  }),
+  })
 );
 
 register.registerMetric(
@@ -112,7 +115,7 @@ register.registerMetric(
       const metric = await botBuffer.getBufferSize();
       this.set(metric);
     },
-  }),
+  })
 );
 
 register.registerMetric(
@@ -123,7 +126,7 @@ register.registerMetric(
       const metric = await sessionBuffer.getBufferSize();
       this.set(metric);
     },
-  }),
+  })
 );
 
 register.registerMetric(
@@ -134,5 +137,5 @@ register.registerMetric(
       const metric = await replayBuffer.getBufferSize();
       this.set(metric);
     },
-  }),
+  })
 );

apps/worker/src/utils/session-handler.ts

@@ -1,13 +1,39 @@
 import type { IServiceCreateEventPayload } from '@openpanel/db';
-import {
+import { sessionsQueue } from '@openpanel/queue';
-  type EventsQueuePayloadCreateSessionEnd,
-  sessionsQueue,
-} from '@openpanel/queue';
-import type { Job } from 'bullmq';
-import { logger } from './logger';
 
 export const SESSION_TIMEOUT = 1000 * 60 * 30;
 
+const CHANGE_DELAY_THROTTLE_MS = process.env.CHANGE_DELAY_THROTTLE_MS
+  ? Number.parseInt(process.env.CHANGE_DELAY_THROTTLE_MS, 10)
+  : 60_000; // 1 minute
+
+const CHANGE_DELAY_THROTTLE_MAP = new Map<string, number>();
+
+export async function extendSessionEndJob({
+  projectId,
+  deviceId,
+}: {
+  projectId: string;
+  deviceId: string;
+}) {
+  const last = CHANGE_DELAY_THROTTLE_MAP.get(`${projectId}:${deviceId}`) ?? 0;
+  const isThrottled = Date.now() - last < CHANGE_DELAY_THROTTLE_MS;
+
+  if (isThrottled) {
+    return;
+  }
+
+  const jobId = getSessionEndJobId(projectId, deviceId);
+  const job = await sessionsQueue.getJob(jobId);
+
+  if (!job) {
+    return;
+  }
+
+  await job.changeDelay(SESSION_TIMEOUT);
+  CHANGE_DELAY_THROTTLE_MAP.set(`${projectId}:${deviceId}`, Date.now());
+}
+
 const getSessionEndJobId = (projectId: string, deviceId: string) =>
   `sessionEnd:${projectId}:${deviceId}`;
 
@@ -33,106 +59,3 @@ export function createSessionEndJob({
     }
   );
 }
-
-export async function getSessionEnd({
-  projectId,
-  deviceId,
-  profileId,
-}: {
-  projectId: string;
-  deviceId: string;
-  profileId: string;
-}) {
-  const sessionEnd = await getSessionEndJob({
-    projectId,
-    deviceId,
-  });
-
-  if (sessionEnd) {
-    const existingSessionIsAnonymous =
-      sessionEnd.job.data.payload.profileId ===
-      sessionEnd.job.data.payload.deviceId;
-
-    const eventIsIdentified =
-      profileId && sessionEnd.job.data.payload.profileId !== profileId;
-
-    if (existingSessionIsAnonymous && eventIsIdentified) {
-      await sessionEnd.job.updateData({
-        ...sessionEnd.job.data,
-        payload: {
-          ...sessionEnd.job.data.payload,
-          profileId,
-        },
-      });
-    }
-
-    await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
-    return sessionEnd.job.data.payload;
-  }
-
-  return null;
-}
-
-export async function getSessionEndJob(args: {
-  projectId: string;
-  deviceId: string;
-  retryCount?: number;
-}): Promise<{
-  deviceId: string;
-  job: Job<EventsQueuePayloadCreateSessionEnd>;
-} | null> {
-  const { retryCount = 0 } = args;
-
-  if (retryCount >= 6) {
-    throw new Error('Failed to get session end');
-  }
-
-  async function handleJobStates(
-    job: Job<EventsQueuePayloadCreateSessionEnd>,
-    deviceId: string
-  ): Promise<{
-    deviceId: string;
-    job: Job<EventsQueuePayloadCreateSessionEnd>;
-  } | null> {
-    const state = await job.getState();
-    if (state !== 'delayed') {
-      logger.debug(`[session-handler] Session end job is in "${state}" state`, {
-        state,
-        retryCount,
-        jobTimestamp: new Date(job.timestamp).toISOString(),
-        jobDelta: Date.now() - job.timestamp,
-        jobId: job.id,
-        payload: job.data.payload,
-      });
-    }
-
-    if (state === 'delayed' || state === 'waiting') {
-      return { deviceId, job };
-    }
-
-    if (state === 'active') {
-      await new Promise((resolve) => setTimeout(resolve, 100));
-      return getSessionEndJob({
-        ...args,
-        retryCount: retryCount + 1,
-      });
-    }
-
-    if (state === 'completed') {
-      await job.remove();
-    }
-
-    return null;
-  }
-
-  // Check current device job
-  const currentJob = await sessionsQueue.getJob(
-    getSessionEndJobId(args.projectId, args.deviceId)
-  );
-  if (currentJob) {
-    return await handleJobStates(currentJob, args.deviceId);
-  }
-
-  // Create session
-  return null;
-}

biome.json

@@ -42,6 +42,7 @@
         "useSemanticElements": "off"
       },
       "style": {
+        "noNestedTernary": "off",
         "noNonNullAssertion": "off",
         "noParameterAssign": "error",
         "useAsConstAssertion": "error",
@@ -70,7 +71,8 @@
         "noDangerouslySetInnerHtml": "off"
       },
       "complexity": {
-        "noForEach": "off"
+        "noForEach": "off",
+        "noExcessiveCognitiveComplexity": "off"
       }
     }
   },

packages/db/src/buffers/event-buffer.test.ts

@@ -2,42 +2,8 @@ import { getRedisCache } from '@openpanel/redis';
 import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { ch } from '../clickhouse/client';
 
-// Mock transformEvent to avoid circular dependency with buffers -> services -> buffers
+// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
-vi.mock('../services/event.service', () => ({
+vi.mock('../services/event.service', () => ({}));
-  transformEvent: (event: any) => ({
-    id: event.id ?? 'id',
-    name: event.name,
-    deviceId: event.device_id,
-    profileId: event.profile_id,
-    projectId: event.project_id,
-    sessionId: event.session_id,
-    properties: event.properties ?? {},
-    createdAt: new Date(event.created_at ?? Date.now()),
-    country: event.country,
-    city: event.city,
-    region: event.region,
-    longitude: event.longitude,
-    latitude: event.latitude,
-    os: event.os,
-    osVersion: event.os_version,
-    browser: event.browser,
-    browserVersion: event.browser_version,
-    device: event.device,
-    brand: event.brand,
-    model: event.model,
-    duration: event.duration ?? 0,
-    path: event.path ?? '',
-    origin: event.origin ?? '',
-    referrer: event.referrer,
-    referrerName: event.referrer_name,
-    referrerType: event.referrer_type,
-    meta: event.meta,
-    importedAt: undefined,
-    sdkName: event.sdk_name,
-    sdkVersion: event.sdk_version,
-    profile: event.profile,
-  }),
-}));
 
 import { EventBuffer } from './event-buffer';
 
@@ -68,18 +34,16 @@ describe('EventBuffer', () => {
       created_at: new Date().toISOString(),
     } as any;
 
-    // Get initial count
     const initialCount = await eventBuffer.getBufferSize();
 
-    // Add event
+    eventBuffer.add(event);
-    await eventBuffer.add(event);
+    await eventBuffer.flush();
 
-    // Buffer counter should increase by 1
     const newCount = await eventBuffer.getBufferSize();
     expect(newCount).toBe(initialCount + 1);
   });
 
-  it('adds multiple screen_views - moves previous to buffer with duration', async () => {
+  it('adds screen_view directly to buffer queue', async () => {
     const t0 = Date.now();
     const sessionId = 'session_1';
 
@@ -99,60 +63,23 @@ describe('EventBuffer', () => {
       created_at: new Date(t0 + 1000).toISOString(),
     } as any;
 
-    const view3 = {
-      project_id: 'p1',
-      profile_id: 'u1',
-      session_id: sessionId,
-      name: 'screen_view',
-      created_at: new Date(t0 + 3000).toISOString(),
-    } as any;
-
-    // Add first screen_view
     const count1 = await eventBuffer.getBufferSize();
-    await eventBuffer.add(view1);
 
-    // Should be stored as "last" but NOT in queue yet
+    eventBuffer.add(view1);
+    await eventBuffer.flush();
+
+    // screen_view goes directly to buffer
     const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1); // No change in buffer
+    expect(count2).toBe(count1 + 1);
 
-    // Last screen_view should be retrievable
+    eventBuffer.add(view2);
-    const last1 = await eventBuffer.getLastScreenView({
+    await eventBuffer.flush();
-      projectId: 'p1',
-      sessionId: sessionId,
-    });
-    expect(last1).not.toBeNull();
-    expect(last1!.createdAt.toISOString()).toBe(view1.created_at);
 
-    // Add second screen_view
-    await eventBuffer.add(view2);
-
-    // Now view1 should be in buffer
     const count3 = await eventBuffer.getBufferSize();
-    expect(count3).toBe(count1 + 1);
+    expect(count3).toBe(count1 + 2);
-
-    // view2 should now be the "last"
-    const last2 = await eventBuffer.getLastScreenView({
-      projectId: 'p1',
-      sessionId: sessionId,
-    });
-    expect(last2!.createdAt.toISOString()).toBe(view2.created_at);
-
-    // Add third screen_view
-    await eventBuffer.add(view3);
-
-    // Now view2 should also be in buffer
-    const count4 = await eventBuffer.getBufferSize();
-    expect(count4).toBe(count1 + 2);
-
-    // view3 should now be the "last"
-    const last3 = await eventBuffer.getLastScreenView({
-      projectId: 'p1',
-      sessionId: sessionId,
-    });
-    expect(last3!.createdAt.toISOString()).toBe(view3.created_at);
   });
 
-  it('adds session_end - moves last screen_view and session_end to buffer', async () => {
+  it('adds session_end directly to buffer queue', async () => {
     const t0 = Date.now();
     const sessionId = 'session_2';
 
@@ -172,148 +99,44 @@ describe('EventBuffer', () => {
       created_at: new Date(t0 + 5000).toISOString(),
     } as any;
 
-    // Add screen_view
     const count1 = await eventBuffer.getBufferSize();
-    await eventBuffer.add(view);
 
-    // Should be stored as "last", not in buffer yet
+    eventBuffer.add(view);
+    eventBuffer.add(sessionEnd);
+    await eventBuffer.flush();
+
     const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1);
+    expect(count2).toBe(count1 + 2);
-
-    // Add session_end
-    await eventBuffer.add(sessionEnd);
-
-    // Both should now be in buffer (+2)
-    const count3 = await eventBuffer.getBufferSize();
-    expect(count3).toBe(count1 + 2);
-
-    // Last screen_view should be cleared
-    const last = await eventBuffer.getLastScreenView({
-      projectId: 'p2',
-      sessionId: sessionId,
-    });
-    expect(last).toBeNull();
-  });
-
-  it('session_end with no previous screen_view - only adds session_end to buffer', async () => {
-    const sessionId = 'session_3';
-
-    const sessionEnd = {
-      project_id: 'p3',
-      profile_id: 'u3',
-      session_id: sessionId,
-      name: 'session_end',
-      created_at: new Date().toISOString(),
-    } as any;
-
-    const count1 = await eventBuffer.getBufferSize();
-    await eventBuffer.add(sessionEnd);
-
-    // Only session_end should be in buffer (+1)
-    const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1 + 1);
-  });
-
-  it('gets last screen_view by profileId', async () => {
-    const view = {
-      project_id: 'p4',
-      profile_id: 'u4',
-      session_id: 'session_4',
-      name: 'screen_view',
-      path: '/home',
-      created_at: new Date().toISOString(),
-    } as any;
-
-    await eventBuffer.add(view);
-
-    // Query by profileId
-    const result = await eventBuffer.getLastScreenView({
-      projectId: 'p4',
-      profileId: 'u4',
-    });
-
-    expect(result).not.toBeNull();
-    expect(result!.name).toBe('screen_view');
-    expect(result!.path).toBe('/home');
-  });
-
-  it('gets last screen_view by sessionId', async () => {
-    const sessionId = 'session_5';
-    const view = {
-      project_id: 'p5',
-      profile_id: 'u5',
-      session_id: sessionId,
-      name: 'screen_view',
-      path: '/about',
-      created_at: new Date().toISOString(),
-    } as any;
-
-    await eventBuffer.add(view);
-
-    // Query by sessionId
-    const result = await eventBuffer.getLastScreenView({
-      projectId: 'p5',
-      sessionId: sessionId,
-    });
-
-    expect(result).not.toBeNull();
-    expect(result!.name).toBe('screen_view');
-    expect(result!.path).toBe('/about');
-  });
-
-  it('returns null for non-existent last screen_view', async () => {
-    const result = await eventBuffer.getLastScreenView({
-      projectId: 'p_nonexistent',
-      profileId: 'u_nonexistent',
-    });
-
-    expect(result).toBeNull();
   });
 
   it('gets buffer count correctly', async () => {
-    // Initially 0
     expect(await eventBuffer.getBufferSize()).toBe(0);
 
-    // Add regular event
+    eventBuffer.add({
-    await eventBuffer.add({
       project_id: 'p6',
       name: 'event1',
       created_at: new Date().toISOString(),
     } as any);
-
+    await eventBuffer.flush();
     expect(await eventBuffer.getBufferSize()).toBe(1);
 
-    // Add another regular event
+    eventBuffer.add({
-    await eventBuffer.add({
       project_id: 'p6',
       name: 'event2',
       created_at: new Date().toISOString(),
     } as any);
-
+    await eventBuffer.flush();
     expect(await eventBuffer.getBufferSize()).toBe(2);
 
-    // Add screen_view (not counted until flushed)
+    // screen_view also goes directly to buffer
-    await eventBuffer.add({
+    eventBuffer.add({
       project_id: 'p6',
       profile_id: 'u6',
       session_id: 'session_6',
       name: 'screen_view',
       created_at: new Date().toISOString(),
     } as any);
-
+    await eventBuffer.flush();
-    // Still 2 (screen_view is pending)
-    expect(await eventBuffer.getBufferSize()).toBe(2);
-
-    // Add another screen_view (first one gets flushed)
-    await eventBuffer.add({
-      project_id: 'p6',
-      profile_id: 'u6',
-      session_id: 'session_6',
-      name: 'screen_view',
-      created_at: new Date(Date.now() + 1000).toISOString(),
-    } as any);
-
-    // Now 3 (2 regular + 1 flushed screen_view)
     expect(await eventBuffer.getBufferSize()).toBe(3);
   });
 
@@ -330,8 +153,9 @@ describe('EventBuffer', () => {
       created_at: new Date(Date.now() + 1000).toISOString(),
     } as any;
 
-    await eventBuffer.add(event1);
+    eventBuffer.add(event1);
-    await eventBuffer.add(event2);
+    eventBuffer.add(event2);
+    await eventBuffer.flush();
 
     expect(await eventBuffer.getBufferSize()).toBe(2);
 
@@ -341,14 +165,12 @@ describe('EventBuffer', () => {
 
     await eventBuffer.processBuffer();
 
-    // Should insert both events
     expect(insertSpy).toHaveBeenCalled();
     const callArgs = insertSpy.mock.calls[0]![0];
     expect(callArgs.format).toBe('JSONEachRow');
     expect(callArgs.table).toBe('events');
     expect(Array.isArray(callArgs.values)).toBe(true);
 
-    // Buffer should be empty after processing
     expect(await eventBuffer.getBufferSize()).toBe(0);
 
     insertSpy.mockRestore();
@@ -359,14 +181,14 @@ describe('EventBuffer', () => {
     process.env.EVENT_BUFFER_CHUNK_SIZE = '2';
     const eb = new EventBuffer();
 
-    // Add 4 events
     for (let i = 0; i < 4; i++) {
-      await eb.add({
+      eb.add({
         project_id: 'p8',
         name: `event${i}`,
         created_at: new Date(Date.now() + i).toISOString(),
       } as any);
     }
+    await eb.flush();
 
     const insertSpy = vi
       .spyOn(ch, 'insert')
@@ -374,14 +196,12 @@ describe('EventBuffer', () => {
 
     await eb.processBuffer();
 
-    // With chunk size 2 and 4 events, should be called twice
     expect(insertSpy).toHaveBeenCalledTimes(2);
     const call1Values = insertSpy.mock.calls[0]![0].values as any[];
     const call2Values = insertSpy.mock.calls[1]![0].values as any[];
     expect(call1Values.length).toBe(2);
     expect(call2Values.length).toBe(2);
 
-    // Restore
     if (prev === undefined) delete process.env.EVENT_BUFFER_CHUNK_SIZE;
     else process.env.EVENT_BUFFER_CHUNK_SIZE = prev;
 
@@ -396,129 +216,61 @@ describe('EventBuffer', () => {
       created_at: new Date().toISOString(),
     } as any;
 
-    await eventBuffer.add(event);
+    eventBuffer.add(event);
+    await eventBuffer.flush();
 
     const count = await eventBuffer.getActiveVisitorCount('p9');
     expect(count).toBeGreaterThanOrEqual(1);
   });
 
-  it('handles multiple sessions independently', async () => {
+  it('handles multiple sessions independently — all events go to buffer', async () => {
     const t0 = Date.now();
+    const count1 = await eventBuffer.getBufferSize();
 
-    // Session 1
+    eventBuffer.add({
-    const view1a = {
       project_id: 'p10',
       profile_id: 'u10',
       session_id: 'session_10a',
       name: 'screen_view',
       created_at: new Date(t0).toISOString(),
-    } as any;
+    } as any);
-
+    eventBuffer.add({
-    const view1b = {
-      project_id: 'p10',
-      profile_id: 'u10',
-      session_id: 'session_10a',
-      name: 'screen_view',
-      created_at: new Date(t0 + 1000).toISOString(),
-    } as any;
-
-    // Session 2
-    const view2a = {
       project_id: 'p10',
       profile_id: 'u11',
       session_id: 'session_10b',
       name: 'screen_view',
       created_at: new Date(t0).toISOString(),
-    } as any;
+    } as any);
-
+    eventBuffer.add({
-    const view2b = {
+      project_id: 'p10',
+      profile_id: 'u10',
+      session_id: 'session_10a',
+      name: 'screen_view',
+      created_at: new Date(t0 + 1000).toISOString(),
+    } as any);
+    eventBuffer.add({
       project_id: 'p10',
       profile_id: 'u11',
       session_id: 'session_10b',
       name: 'screen_view',
       created_at: new Date(t0 + 2000).toISOString(),
-    } as any;
+    } as any);
+    await eventBuffer.flush();
 
-    await eventBuffer.add(view1a);
+    // All 4 events are in buffer directly
-    await eventBuffer.add(view2a);
+    expect(await eventBuffer.getBufferSize()).toBe(count1 + 4);
-    await eventBuffer.add(view1b); // Flushes view1a
-    await eventBuffer.add(view2b); // Flushes view2a
-
-    // Should have 2 events in buffer (one from each session)
-    expect(await eventBuffer.getBufferSize()).toBe(2);
-
-    // Each session should have its own "last" screen_view
-    const last1 = await eventBuffer.getLastScreenView({
-      projectId: 'p10',
-      sessionId: 'session_10a',
-    });
-    expect(last1!.createdAt.toISOString()).toBe(view1b.created_at);
-
-    const last2 = await eventBuffer.getLastScreenView({
-      projectId: 'p10',
-      sessionId: 'session_10b',
-    });
-    expect(last2!.createdAt.toISOString()).toBe(view2b.created_at);
   });
 
-  it('screen_view without session_id goes directly to buffer', async () => {
+  it('bulk adds events to buffer', async () => {
-    const view = {
+    const events = Array.from({ length: 5 }, (_, i) => ({
       project_id: 'p11',
-      profile_id: 'u11',
+      name: `event${i}`,
-      name: 'screen_view',
+      created_at: new Date(Date.now() + i).toISOString(),
-      created_at: new Date().toISOString(),
+    })) as any[];
-    } as any;
 
-    const count1 = await eventBuffer.getBufferSize();
+    eventBuffer.bulkAdd(events);
-    await eventBuffer.add(view);
+    await eventBuffer.flush();
 
-    // Should go directly to buffer (no session_id)
+    expect(await eventBuffer.getBufferSize()).toBe(5);
-    const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1 + 1);
-  });
-
-  it('updates last screen_view when new one arrives from same profile but different session', async () => {
-    const t0 = Date.now();
-
-    const view1 = {
-      project_id: 'p12',
-      profile_id: 'u12',
-      session_id: 'session_12a',
-      name: 'screen_view',
-      path: '/page1',
-      created_at: new Date(t0).toISOString(),
-    } as any;
-
-    const view2 = {
-      project_id: 'p12',
-      profile_id: 'u12',
-      session_id: 'session_12b', // Different session!
-      name: 'screen_view',
-      path: '/page2',
-      created_at: new Date(t0 + 1000).toISOString(),
-    } as any;
-
-    await eventBuffer.add(view1);
-    await eventBuffer.add(view2);
-
-    // Both sessions should have their own "last"
-    const lastSession1 = await eventBuffer.getLastScreenView({
-      projectId: 'p12',
-      sessionId: 'session_12a',
-    });
-    expect(lastSession1!.path).toBe('/page1');
-
-    const lastSession2 = await eventBuffer.getLastScreenView({
-      projectId: 'p12',
-      sessionId: 'session_12b',
-    });
-    expect(lastSession2!.path).toBe('/page2');
-
-    // Profile should have the latest one
-    const lastProfile = await eventBuffer.getLastScreenView({
-      projectId: 'p12',
-      profileId: 'u12',
-    });
-    expect(lastProfile!.path).toBe('/page2');
   });
 });

packages/db/src/buffers/event-buffer.ts

@@ -2,33 +2,13 @@ import { getSafeJson } from '@openpanel/json';
 import {
   type Redis,
   getRedisCache,
-  getRedisPub,
   publishEvent,
 } from '@openpanel/redis';
 import { ch } from '../clickhouse/client';
-import {
+import { type IClickhouseEvent } from '../services/event.service';
-  type IClickhouseEvent,
-  type IServiceEvent,
-  transformEvent,
-} from '../services/event.service';
 import { BaseBuffer } from './base-buffer';
 
-/**
- * Simplified Event Buffer
- *
- * Rules:
- * 1. All events go into a single list buffer (event_buffer:queue)
- * 2. screen_view events are handled specially:
- *    - Store current screen_view as "last" for the session
- *    - When a new screen_view arrives, flush the previous one with calculated duration
- * 3. session_end events:
- *    - Retrieve the last screen_view (don't modify it)
- *    - Push both screen_view and session_end to buffer
- * 4. Flush: Simply process all events from the list buffer
- */
-
 export class EventBuffer extends BaseBuffer {
-  // Configurable limits
   private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE
     ? Number.parseInt(process.env.EVENT_BUFFER_BATCH_SIZE, 10)
     : 4000;
@@ -36,124 +16,26 @@ export class EventBuffer extends BaseBuffer {
     ? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)
     : 1000;
 
+  private microBatchIntervalMs = process.env.EVENT_BUFFER_MICRO_BATCH_MS
+    ? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_MS, 10)
+    : 10;
+  private microBatchMaxSize = process.env.EVENT_BUFFER_MICRO_BATCH_SIZE
+    ? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_SIZE, 10)
+    : 100;
+
+  private pendingEvents: IClickhouseEvent[] = [];
+  private flushTimer: ReturnType<typeof setTimeout> | null = null;
+  private isFlushing = false;
+  /** Tracks consecutive flush failures for observability; reset on success. */
+  private flushRetryCount = 0;
+
   private activeVisitorsExpiration = 60 * 5; // 5 minutes
-
+  /** How often (ms) we refresh the heartbeat key + zadd per visitor. */
-  // LIST - Stores all events ready to be flushed
+  private heartbeatRefreshMs = 60_000; // 1 minute
+  private lastHeartbeat = new Map<string, number>();
   private queueKey = 'event_buffer:queue';
-
-  // STRING - Tracks total buffer size incrementally
   protected bufferCounterKey = 'event_buffer:total_count';
 
-  // Script SHAs for loaded Lua scripts
-  private scriptShas: {
-    addScreenView?: string;
-    addSessionEnd?: string;
-  } = {};
-
-  // Hash key for storing last screen_view per session
-  private getLastScreenViewKeyBySession(sessionId: string) {
-    return `event_buffer:last_screen_view:session:${sessionId}`;
-  }
-
-  // Hash key for storing last screen_view per profile
-  private getLastScreenViewKeyByProfile(projectId: string, profileId: string) {
-    return `event_buffer:last_screen_view:profile:${projectId}:${profileId}`;
-  }
-
-  /**
-   * Lua script for handling screen_view addition - RACE-CONDITION SAFE without GroupMQ
-   *
-   * Strategy: Use Redis GETDEL (atomic get-and-delete) to ensure only ONE thread
-   * can process the "last" screen_view at a time.
-   *
-   * KEYS[1] = last screen_view key (by session) - stores both event and timestamp as JSON
-   * KEYS[2] = last screen_view key (by profile, may be empty)
-   * KEYS[3] = queue key
-   * KEYS[4] = buffer counter key
-   * ARGV[1] = new event with timestamp as JSON: {"event": {...}, "ts": 123456}
-   * ARGV[2] = TTL for last screen_view (1 hour)
-   */
-  private readonly addScreenViewScript = `
-local sessionKey = KEYS[1]
-local profileKey = KEYS[2]
-local queueKey = KEYS[3]
-local counterKey = KEYS[4]
-local newEventData = ARGV[1]
-local ttl = tonumber(ARGV[2])
-
--- GETDEL is atomic: get previous and delete in one operation
--- This ensures only ONE thread gets the previous event
-local previousEventData = redis.call("GETDEL", sessionKey)
-
--- Store new screen_view as last for session
-redis.call("SET", sessionKey, newEventData, "EX", ttl)
-
--- Store new screen_view as last for profile (if key provided)
-if profileKey and profileKey ~= "" then
-  redis.call("SET", profileKey, newEventData, "EX", ttl)
-end
-
--- If there was a previous screen_view, add it to queue with calculated duration
-if previousEventData then
-  local prev = cjson.decode(previousEventData)
-  local curr = cjson.decode(newEventData)
-  
-  -- Calculate duration (ensure non-negative to handle clock skew)
-  if prev.ts and curr.ts then
-    prev.event.duration = math.max(0, curr.ts - prev.ts)
-  end
-  
-  redis.call("RPUSH", queueKey, cjson.encode(prev.event))
-  redis.call("INCR", counterKey)
-  return 1
-end
-
-return 0
-`;
-
-  /**
-   * Lua script for handling session_end - RACE-CONDITION SAFE
-   *
-   * Uses GETDEL to atomically retrieve and delete the last screen_view
-   *
-   * KEYS[1] = last screen_view key (by session)
-   * KEYS[2] = last screen_view key (by profile, may be empty)
-   * KEYS[3] = queue key
-   * KEYS[4] = buffer counter key
-   * ARGV[1] = session_end event JSON
-   */
-  private readonly addSessionEndScript = `
-local sessionKey = KEYS[1]
-local profileKey = KEYS[2]
-local queueKey = KEYS[3]
-local counterKey = KEYS[4]
-local sessionEndJson = ARGV[1]
-
--- GETDEL is atomic: only ONE thread gets the last screen_view
-local previousEventData = redis.call("GETDEL", sessionKey)
-local added = 0
-
--- If there was a previous screen_view, add it to queue
-if previousEventData then
-  local prev = cjson.decode(previousEventData)
-  redis.call("RPUSH", queueKey, cjson.encode(prev.event))
-  redis.call("INCR", counterKey)
-  added = added + 1
-end
-
--- Add session_end to queue
-redis.call("RPUSH", queueKey, sessionEndJson)
-redis.call("INCR", counterKey)
-added = added + 1
-
--- Delete profile key
-if profileKey and profileKey ~= "" then
-  redis.call("DEL", profileKey)
-end
-
-return added
-`;
-
   constructor() {
     super({
       name: 'event',
@@ -161,170 +43,97 @@ return added
         await this.processBuffer();
       },
     });
-    // Load Lua scripts into Redis on startup
-    this.loadScripts();
-  }
-
-  /**
-   * Load Lua scripts into Redis and cache their SHAs.
-   * This avoids sending the entire script on every call.
-   */
-  private async loadScripts() {
-    try {
-      const redis = getRedisCache();
-      const [screenViewSha, sessionEndSha] = await Promise.all([
-        redis.script('LOAD', this.addScreenViewScript),
-        redis.script('LOAD', this.addSessionEndScript),
-      ]);
-
-      this.scriptShas.addScreenView = screenViewSha as string;
-      this.scriptShas.addSessionEnd = sessionEndSha as string;
-
-      this.logger.info('Loaded Lua scripts into Redis', {
-        addScreenView: this.scriptShas.addScreenView,
-        addSessionEnd: this.scriptShas.addSessionEnd,
-      });
-    } catch (error) {
-      this.logger.error('Failed to load Lua scripts', { error });
-    }
   }
 
   bulkAdd(events: IClickhouseEvent[]) {
-    const redis = getRedisCache();
-    const multi = redis.multi();
     for (const event of events) {
-      this.add(event, multi);
+      this.add(event);
     }
-    return multi.exec();
   }
 
-  /**
+  add(event: IClickhouseEvent) {
-   * Add an event into Redis buffer.
+    this.pendingEvents.push(event);
-   *
+
-   * Logic:
+    if (this.pendingEvents.length >= this.microBatchMaxSize) {
-   * - screen_view: Store as "last" for session, flush previous if exists
+      this.flushLocalBuffer();
-   * - session_end: Flush last screen_view + session_end
+      return;
-   * - Other events: Add directly to queue
+    }
-   */
+
-  async add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
+    if (!this.flushTimer) {
+      this.flushTimer = setTimeout(() => {
+        this.flushTimer = null;
+        this.flushLocalBuffer();
+      }, this.microBatchIntervalMs);
+    }
+  }
+
+  public async flush() {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    await this.flushLocalBuffer();
+  }
+
+  private async flushLocalBuffer() {
+    if (this.isFlushing || this.pendingEvents.length === 0) {
+      return;
+    }
+
+    this.isFlushing = true;
+
+    const eventsToFlush = this.pendingEvents;
+    this.pendingEvents = [];
+
     try {
       const redis = getRedisCache();
-      const eventJson = JSON.stringify(event);
+      const multi = redis.multi();
-      const multi = _multi || redis.multi();
 
-      if (event.session_id && event.name === 'screen_view') {
+      for (const event of eventsToFlush) {
-        // Handle screen_view
+        multi.rpush(this.queueKey, JSON.stringify(event));
-        const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
+        if (event.profile_id) {
-        const profileKey = event.profile_id
+          this.incrementActiveVisitorCount(
-          ? this.getLastScreenViewKeyByProfile(
+            multi,
-              event.project_id,
+            event.project_id,
-              event.profile_id,
+            event.profile_id,
-            )
+          );
-          : '';
+        }
-        const timestamp = new Date(event.created_at || Date.now()).getTime();
-
-        // Combine event and timestamp into single JSON for atomic operations
-        const eventWithTimestamp = JSON.stringify({
-          event: event,
-          ts: timestamp,
-        });
-
-        this.evalScript(
-          multi,
-          'addScreenView',
-          this.addScreenViewScript,
-          4,
-          sessionKey,
-          profileKey,
-          this.queueKey,
-          this.bufferCounterKey,
-          eventWithTimestamp,
-          '3600', // 1 hour TTL
-        );
-      } else if (event.session_id && event.name === 'session_end') {
-        // Handle session_end
-        const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
-        const profileKey = event.profile_id
-          ? this.getLastScreenViewKeyByProfile(
-              event.project_id,
-              event.profile_id,
-            )
-          : '';
-
-        this.evalScript(
-          multi,
-          'addSessionEnd',
-          this.addSessionEndScript,
-          4,
-          sessionKey,
-          profileKey,
-          this.queueKey,
-          this.bufferCounterKey,
-          eventJson,
-        );
-      } else {
-        // All other events go directly to queue
-        multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
       }
+      multi.incrby(this.bufferCounterKey, eventsToFlush.length);
 
-      if (event.profile_id) {
+      await multi.exec();
-        this.incrementActiveVisitorCount(
-          multi,
-          event.project_id,
-          event.profile_id,
-        );
-      }
 
-      if (!_multi) {
+      this.flushRetryCount = 0;
-        await multi.exec();
+      this.pruneHeartbeatMap();
-      }
-
-      await publishEvent('events', 'received', transformEvent(event));
     } catch (error) {
-      this.logger.error('Failed to add event to Redis buffer', { error });
+      // Re-queue failed events at the front to preserve order and avoid data loss
+      this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
+
+      this.flushRetryCount += 1;
+      this.logger.warn(
+        'Failed to flush local buffer to Redis; events re-queued',
+        {
+          error,
+          eventCount: eventsToFlush.length,
+          flushRetryCount: this.flushRetryCount,
+        },
+      );
+    } finally {
+      this.isFlushing = false;
+      // Events may have accumulated while we were flushing; schedule another flush if needed
+      if (this.pendingEvents.length > 0 && !this.flushTimer) {
+        this.flushTimer = setTimeout(() => {
+          this.flushTimer = null;
+          this.flushLocalBuffer();
+        }, this.microBatchIntervalMs);
+      }
     }
   }
 
-  /**
-   * Execute a Lua script using EVALSHA (cached) or fallback to EVAL.
-   * This avoids sending the entire script on every call.
-   */
-  private evalScript(
-    multi: ReturnType<Redis['multi']>,
-    scriptName: keyof typeof this.scriptShas,
-    scriptContent: string,
-    numKeys: number,
-    ...args: (string | number)[]
-  ) {
-    const sha = this.scriptShas[scriptName];
-
-    if (sha) {
-      // Use EVALSHA with cached SHA
-      multi.evalsha(sha, numKeys, ...args);
-    } else {
-      // Fallback to EVAL and try to reload script
-      multi.eval(scriptContent, numKeys, ...args);
-      this.logger.warn(`Script ${scriptName} not loaded, using EVAL fallback`);
-      // Attempt to reload scripts in background
-      this.loadScripts();
-    }
-  }
-
-  /**
-   * Process the Redis buffer - simplified version.
-   *
-   * Simply:
-   * 1. Fetch events from the queue (up to batchSize)
-   * 2. Parse and sort them
-   * 3. Insert into ClickHouse in chunks
-   * 4. Publish saved events
-   * 5. Clean up processed events from queue
-   */
   async processBuffer() {
     const redis = getRedisCache();
 
     try {
-      // Fetch events from queue
       const queueEvents = await redis.lrange(
         this.queueKey,
         0,
@@ -336,7 +145,6 @@ return added
         return;
       }
 
-      // Parse events
       const eventsToClickhouse: IClickhouseEvent[] = [];
       for (const eventStr of queueEvents) {
         const event = getSafeJson<IClickhouseEvent>(eventStr);
@@ -350,14 +158,12 @@ return added
         return;
       }
 
-      // Sort events by creation time
       eventsToClickhouse.sort(
         (a, b) =>
           new Date(a.created_at || 0).getTime() -
           new Date(b.created_at || 0).getTime(),
       );
 
-      // Insert events into ClickHouse in chunks
       this.logger.info('Inserting events into ClickHouse', {
         totalEvents: eventsToClickhouse.length,
         chunks: Math.ceil(eventsToClickhouse.length / this.chunkSize),
@@ -371,14 +177,17 @@ return added
         });
       }
 
-      // Publish "saved" events
+      const countByProject = new Map<string, number>();
-      const pubMulti = getRedisPub().multi();
       for (const event of eventsToClickhouse) {
-        await publishEvent('events', 'saved', transformEvent(event), pubMulti);
+        countByProject.set(
+          event.project_id,
+          (countByProject.get(event.project_id) ?? 0) + 1,
+        );
+      }
+      for (const [projectId, count] of countByProject) {
+        publishEvent('events', 'batch', { projectId, count });
       }
-      await pubMulti.exec();
 
-      // Clean up processed events from queue
       await redis
         .multi()
         .ltrim(this.queueKey, queueEvents.length, -1)
@@ -394,45 +203,6 @@ return added
     }
   }
 
-  /**
-   * Retrieve the latest screen_view event for a given session or profile
-   */
-  public async getLastScreenView(
-    params:
-      | {
-          sessionId: string;
-        }
-      | {
-          projectId: string;
-          profileId: string;
-        },
-  ): Promise<IServiceEvent | null> {
-    const redis = getRedisCache();
-
-    let lastScreenViewKey: string;
-    if ('sessionId' in params) {
-      lastScreenViewKey = this.getLastScreenViewKeyBySession(params.sessionId);
-    } else {
-      lastScreenViewKey = this.getLastScreenViewKeyByProfile(
-        params.projectId,
-        params.profileId,
-      );
-    }
-
-    const eventDataStr = await redis.get(lastScreenViewKey);
-
-    if (eventDataStr) {
-      const eventData = getSafeJson<{ event: IClickhouseEvent; ts: number }>(
-        eventDataStr,
-      );
-      if (eventData?.event) {
-        return transformEvent(eventData.event);
-      }
-    }
-
-    return null;
-  }
-
   public async getBufferSize() {
     return this.getBufferSizeWithCounter(async () => {
       const redis = getRedisCache();
@@ -440,16 +210,32 @@ return added
     });
   }
 
-  private async incrementActiveVisitorCount(
+  private pruneHeartbeatMap() {
+    const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
+    for (const [key, ts] of this.lastHeartbeat) {
+      if (ts < cutoff) {
+        this.lastHeartbeat.delete(key);
+      }
+    }
+  }
+
+  private incrementActiveVisitorCount(
     multi: ReturnType<Redis['multi']>,
     projectId: string,
     profileId: string,
   ) {
-    // Track active visitors and emit expiry events when inactive for TTL
+    const key = `${projectId}:${profileId}`;
     const now = Date.now();
+    const last = this.lastHeartbeat.get(key) ?? 0;
+
+    if (now - last < this.heartbeatRefreshMs) {
+      return;
+    }
+
+    this.lastHeartbeat.set(key, now);
     const zsetKey = `live:visitors:${projectId}`;
     const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
-    return multi
+    multi
       .zadd(zsetKey, now, profileId)
       .set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
   }

packages/db/src/buffers/profile-buffer.ts

@@ -1,11 +1,11 @@
 import { deepMergeObjects } from '@openpanel/common';
 import { getSafeJson } from '@openpanel/json';
 import type { ILogger } from '@openpanel/logger';
-import { type Redis, getRedisCache } from '@openpanel/redis';
+import { getRedisCache, type Redis } from '@openpanel/redis';
 import shallowEqual from 'fast-deep-equal';
 import { omit } from 'ramda';
 import sqlstring from 'sqlstring';
-import { TABLE_NAMES, ch, chQuery } from '../clickhouse/client';
+import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
 import type { IClickhouseProfile } from '../services/profile.service';
 import { BaseBuffer } from './base-buffer';
 
@@ -89,7 +89,7 @@ export class ProfileBuffer extends BaseBuffer {
             'os_version',
             'browser_version',
           ],
-          profile.properties,
+          profile.properties
         );
       }
 
@@ -97,16 +97,16 @@ export class ProfileBuffer extends BaseBuffer {
         ? deepMergeObjects(existingProfile, omit(['created_at'], profile))
         : profile;
 
-      if (profile && existingProfile) {
+      if (
-        if (
+        profile &&
-          shallowEqual(
+        existingProfile &&
-            omit(['created_at'], existingProfile),
+        shallowEqual(
-            omit(['created_at'], mergedProfile),
+          omit(['created_at'], existingProfile),
-          )
+          omit(['created_at'], mergedProfile)
-        ) {
+        )
-          this.logger.debug('Profile not changed, skipping');
+      ) {
-          return;
+        this.logger.debug('Profile not changed, skipping');
-        }
+        return;
       }
 
       this.logger.debug('Merged profile will be inserted', {
@@ -151,11 +151,11 @@ export class ProfileBuffer extends BaseBuffer {
 
   private async fetchProfile(
     profile: IClickhouseProfile,
-    logger: ILogger,
+    logger: ILogger
   ): Promise<IClickhouseProfile | null> {
     const existingProfile = await this.fetchFromCache(
       profile.id,
-      profile.project_id,
+      profile.project_id
     );
     if (existingProfile) {
       logger.debug('Profile found in Redis');
@@ -167,7 +167,7 @@ export class ProfileBuffer extends BaseBuffer {
 
   public async fetchFromCache(
     profileId: string,
-    projectId: string,
+    projectId: string
   ): Promise<IClickhouseProfile | null> {
     const cacheKey = this.getProfileCacheKey({
       profileId,
@@ -182,7 +182,7 @@ export class ProfileBuffer extends BaseBuffer {
 
   private async fetchFromClickhouse(
     profile: IClickhouseProfile,
-    logger: ILogger,
+    logger: ILogger
   ): Promise<IClickhouseProfile | null> {
     logger.debug('Fetching profile from Clickhouse');
     const result = await chQuery<IClickhouseProfile>(
@@ -207,7 +207,7 @@ export class ProfileBuffer extends BaseBuffer {
         }
       GROUP BY id, project_id 
       ORDER BY created_at DESC 
-      LIMIT 1`,
+      LIMIT 1`
     );
     logger.debug('Clickhouse fetch result', {
       found: !!result[0],
@@ -221,7 +221,7 @@ export class ProfileBuffer extends BaseBuffer {
       const profiles = await this.redis.lrange(
         this.redisKey,
         0,
-        this.batchSize - 1,
+        this.batchSize - 1
       );
 
       if (profiles.length === 0) {
@@ -231,7 +231,7 @@ export class ProfileBuffer extends BaseBuffer {
 
       this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
       const parsedProfiles = profiles.map((p) =>
-        getSafeJson<IClickhouseProfile>(p),
+        getSafeJson<IClickhouseProfile>(p)
       );
 
       for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {

packages/db/src/services/clients.service.ts

@@ -1,4 +1,4 @@
-import { cacheable, cacheableLru } from '@openpanel/redis';
+import { cacheable } from '@openpanel/redis';
 import type { Client, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 
@@ -34,7 +34,4 @@ export async function getClientById(
   });
 }
 
-export const getClientByIdCached = cacheableLru(getClientById, {
+export const getClientByIdCached = cacheable(getClientById, 60 * 5);
-  maxSize: 1000,
-  ttl: 60 * 5,
-});

packages/db/src/services/event.service.ts

@@ -168,7 +168,6 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
     device: event.device,
     brand: event.brand,
     model: event.model,
-    duration: event.duration,
     path: event.path,
     origin: event.origin,
     referrer: event.referrer,
@@ -216,7 +215,7 @@ export interface IServiceEvent {
   device?: string | undefined;
   brand?: string | undefined;
   model?: string | undefined;
-  duration: number;
+  duration?: number;
   path: string;
   origin: string;
   referrer: string | undefined;
@@ -247,7 +246,7 @@ export interface IServiceEventMinimal {
   browser?: string | undefined;
   device?: string | undefined;
   brand?: string | undefined;
-  duration: number;
+  duration?: number;
   path: string;
   origin: string;
   referrer: string | undefined;
@@ -379,7 +378,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
     device: payload.device ?? '',
     brand: payload.brand ?? '',
     model: payload.model ?? '',
-    duration: payload.duration,
+    duration: payload.duration ?? 0,
     referrer: payload.referrer ?? '',
     referrer_name: payload.referrerName ?? '',
     referrer_type: payload.referrerType ?? '',
@@ -477,7 +476,7 @@ export async function getEventList(options: GetEventListOptions) {
     sb.where.cursor = `created_at < ${sqlstring.escape(formatClickhouseDate(cursor))}`;
   }
 
-  if (!cursor && !(startDate && endDate)) {
+  if (!(cursor || (startDate && endDate))) {
     sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;
   }
 
@@ -562,9 +561,6 @@ export async function getEventList(options: GetEventListOptions) {
   if (select.model) {
     sb.select.model = 'model';
   }
-  if (select.duration) {
-    sb.select.duration = 'duration';
-  }
   if (select.path) {
     sb.select.path = 'path';
   }
@@ -771,7 +767,6 @@ class EventService {
     where,
     select,
     limit,
-    orderBy,
     filters,
   }: {
     projectId: string;
@@ -811,7 +806,6 @@ class EventService {
         select.event.deviceId && 'e.device_id as device_id',
         select.event.name && 'e.name as name',
         select.event.path && 'e.path as path',
-        select.event.duration && 'e.duration as duration',
         select.event.country && 'e.country as country',
         select.event.city && 'e.city as city',
         select.event.os && 'e.os as os',
@@ -896,7 +890,6 @@ class EventService {
         select.event.deviceId && 'e.device_id as device_id',
         select.event.name && 'e.name as name',
         select.event.path && 'e.path as path',
-        select.event.duration && 'e.duration as duration',
         select.event.country && 'e.country as country',
         select.event.city && 'e.city as city',
         select.event.os && 'e.os as os',
@@ -1032,7 +1025,6 @@ class EventService {
           id: true,
           name: true,
           createdAt: true,
-          duration: true,
           country: true,
           city: true,
           os: true,

packages/db/src/services/notification.service.ts

@@ -90,7 +90,7 @@ export const getNotificationRulesByProjectId = cacheable(
       },
     });
   },
-  60 * 24
+  60 * 24,
 );
 
 function getIntegration(integrationId: string | null) {

packages/db/src/services/overview.service.ts

@@ -416,6 +416,30 @@ export class OverviewService {
     const where = this.getRawWhereClause('sessions', filters);
     const fillConfig = this.getFillConfig(interval, startDate, endDate);
 
+    // CTE: per-event screen_view durations via window function
+    const rawScreenViewDurationsQuery = clix(this.client, timezone)
+      .select([
+        `${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
+        `dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
+      ])
+      .from(TABLE_NAMES.events)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'screen_view')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ])
+      .rawWhere(this.getRawWhereClause('events', filters));
+
+    // CTE: avg duration per date bucket
+    const avgDurationByDateQuery = clix(this.client, timezone)
+      .select([
+        'date',
+        'round(avgIf(duration, duration > 0), 2) / 1000 AS avg_session_duration',
+      ])
+      .from('raw_screen_view_durations')
+      .groupBy(['date']);
+
     // Session aggregation with bounce rates
     const sessionAggQuery = clix(this.client, timezone)
       .select([
@@ -473,6 +497,8 @@ export class OverviewService {
           .where('date', '!=', rollupDate)
       )
       .with('overall_unique_visitors', overallUniqueVisitorsQuery)
+      .with('raw_screen_view_durations', rawScreenViewDurationsQuery)
+      .with('avg_duration_by_date', avgDurationByDateQuery)
       .select<{
         date: string;
         bounce_rate: number;
@@ -489,8 +515,7 @@ export class OverviewService {
         'dss.bounce_rate as bounce_rate',
         'uniq(e.profile_id) AS unique_visitors',
         'uniq(e.session_id) AS total_sessions',
-        'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
+        'coalesce(dur.avg_session_duration, 0) AS avg_session_duration',
-        'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
         'count(*) AS total_screen_views',
         'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
         '(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
@@ -502,6 +527,10 @@ export class OverviewService {
         'daily_session_stats AS dss',
         `${clix.toStartOf('e.created_at', interval as any)} = dss.date`
       )
+      .leftJoin(
+        'avg_duration_by_date AS dur',
+        `${clix.toStartOf('e.created_at', interval as any)} = dur.date`
+      )
       .where('e.project_id', '=', projectId)
       .where('e.name', '=', 'screen_view')
       .where('e.created_at', 'BETWEEN', [
@@ -509,7 +538,7 @@ export class OverviewService {
         clix.datetime(endDate, 'toDateTime'),
       ])
       .rawWhere(this.getRawWhereClause('events', filters))
-      .groupBy(['date', 'dss.bounce_rate'])
+      .groupBy(['date', 'dss.bounce_rate', 'dur.avg_session_duration'])
       .orderBy('date', 'ASC')
       .fill(fillConfig.from, fillConfig.to, fillConfig.step)
       .transform({

packages/db/src/services/pages.service.ts

@@ -52,6 +52,24 @@ export class PagesService {
       .where('created_at', '>=', clix.exp('now() - INTERVAL 30 DAY'))
       .groupBy(['origin', 'path']);
 
+    // CTE: compute screen_view durations via window function (leadInFrame gives next event's timestamp)
+    const screenViewDurationsCte = clix(this.client, timezone)
+      .select([
+        'project_id',
+        'session_id',
+        'path',
+        'origin',
+        `dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
+      ])
+      .from(TABLE_NAMES.events, false)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'screen_view')
+      .where('path', '!=', '')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ]);
+
     // Pre-filtered sessions subquery for better performance
     const sessionsSubquery = clix(this.client, timezone)
       .select(['id', 'project_id', 'is_bounce'])
@@ -66,6 +84,7 @@ export class PagesService {
     // Main query: aggregate events and calculate bounce rate from pre-filtered sessions
     const query = clix(this.client, timezone)
       .with('page_titles', titlesCte)
+      .with('screen_view_durations', screenViewDurationsCte)
       .select<ITopPage>([
         'e.origin as origin',
         'e.path as path',
@@ -74,25 +93,18 @@ export class PagesService {
         'count() as pageviews',
         'round(avg(e.duration) / 1000 / 60, 2) as avg_duration',
         `round(
-          (uniqIf(e.session_id, s.is_bounce = 1) * 100.0) / 
+          (uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /
-          nullIf(uniq(e.session_id), 0), 
+          nullIf(uniq(e.session_id), 0),
           2
         ) as bounce_rate`,
       ])
-      .from(`${TABLE_NAMES.events} e`, false)
+      .from('screen_view_durations e', false)
       .leftJoin(
         sessionsSubquery,
         'e.session_id = s.id AND e.project_id = s.project_id',
         's'
       )
       .leftJoin('page_titles pt', 'concat(e.origin, e.path) = pt.page_key')
-      .where('e.project_id', '=', projectId)
-      .where('e.name', '=', 'screen_view')
-      .where('e.path', '!=', '')
-      .where('e.created_at', 'BETWEEN', [
-        clix.datetime(startDate, 'toDateTime'),
-        clix.datetime(endDate, 'toDateTime'),
-      ])
       .when(!!search, (q) => {
         const term = `%${search}%`;
         q.whereGroup()

packages/db/src/services/project.service.ts

@@ -1,6 +1,6 @@
 import { cacheable } from '@openpanel/redis';
 import sqlstring from 'sqlstring';
-import { TABLE_NAMES, chQuery } from '../clickhouse/client';
+import { chQuery, TABLE_NAMES } from '../clickhouse/client';
 import type { Prisma, Project } from '../prisma-client';
 import { db } from '../prisma-client';
 
@@ -25,6 +25,7 @@ export async function getProjectById(id: string) {
   return res;
 }
 
+/** L1 LRU (60s) + L2 Redis. clear() invalidates Redis + local LRU; other nodes may serve stale from LRU for up to 60s. */
 export const getProjectByIdCached = cacheable(getProjectById, 60 * 60 * 24);
 
 export async function getProjectWithClients(id: string) {
@@ -44,7 +45,7 @@ export async function getProjectWithClients(id: string) {
   return res;
 }
 
-export async function getProjectsByOrganizationId(organizationId: string) {
+export function getProjectsByOrganizationId(organizationId: string) {
   return db.project.findMany({
     where: {
       organizationId,
@@ -95,7 +96,7 @@ export async function getProjects({
 
   if (access.length > 0) {
     return projects.filter((project) =>
-      access.some((a) => a.projectId === project.id),
+      access.some((a) => a.projectId === project.id)
     );
   }
 
@@ -104,7 +105,7 @@ export async function getProjects({
 
 export const getProjectEventsCount = async (projectId: string) => {
   const res = await chQuery<{ count: number }>(
-    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
+    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
   );
   return res[0]?.count;
 };

packages/db/src/services/salt.service.ts

@@ -1,9 +1,9 @@
 import { generateSalt } from '@openpanel/common/server';
 
-import { cacheableLru } from '@openpanel/redis';
+import { cacheable } from '@openpanel/redis';
 import { db } from '../prisma-client';
 
-export const getSalts = cacheableLru(
+export const getSalts = cacheable(
   'op:salt',
   async () => {
     const [curr, prev] = await db.salt.findMany({
@@ -24,10 +24,7 @@ export const getSalts = cacheableLru(
 
     return salts;
   },
-  {
+  60 * 5,
-    maxSize: 2,
-    ttl: 60 * 5,
-  },
 );
 
 export async function createInitialSalts() {

packages/payments/scripts/assign-product-to-org.ts

@@ -1,221 +0,0 @@
-import { db } from '@openpanel/db';
-import { Polar } from '@polar-sh/sdk';
-import inquirer from 'inquirer';
-import inquirerAutocomplete from 'inquirer-autocomplete-prompt';
-import { getSuccessUrl } from '..';
-
-// Register the autocomplete prompt
-inquirer.registerPrompt('autocomplete', inquirerAutocomplete);
-
-interface Answers {
-  isProduction: boolean;
-  polarApiKey: string;
-  productId: string;
-  organizationId: string;
-}
-
-async function promptForInput() {
-  // Get all organizations first
-  const organizations = await db.organization.findMany({
-    select: {
-      id: true,
-      name: true,
-    },
-  });
-
-  // Step 1: Collect Polar credentials first
-  const polarCredentials = await inquirer.prompt<{
-    isProduction: boolean;
-    polarApiKey: string;
-    polarOrganizationId: string;
-  }>([
-    {
-      type: 'list',
-      name: 'isProduction',
-      message: 'Is this for production?',
-      choices: [
-        { name: 'Yes', value: true },
-        { name: 'No', value: false },
-      ],
-      default: true,
-    },
-    {
-      type: 'string',
-      name: 'polarApiKey',
-      message: 'Enter your Polar API key:',
-      validate: (input: string) => {
-        if (!input) return 'API key is required';
-        return true;
-      },
-    },
-  ]);
-
-  // Step 2: Initialize Polar client and fetch products
-  const polar = new Polar({
-    accessToken: polarCredentials.polarApiKey,
-    server: polarCredentials.isProduction ? 'production' : 'sandbox',
-  });
-
-  console.log('Fetching products from Polar...');
-  const productsResponse = await polar.products.list({
-    limit: 100,
-    isArchived: false,
-    sorting: ['price_amount'],
-  });
-
-  const products = productsResponse.result.items;
-
-  if (products.length === 0) {
-    throw new Error('No products found in Polar');
-  }
-
-  // Step 3: Continue with product selection and organization selection
-  const restOfAnswers = await inquirer.prompt<{
-    productId: string;
-    organizationId: string;
-  }>([
-    {
-      type: 'autocomplete',
-      name: 'productId',
-      message: 'Select product:',
-      source: (answersSoFar: any, input = '') => {
-        return products
-          .filter(
-            (product) =>
-              product.name.toLowerCase().includes(input.toLowerCase()) ||
-              product.id.toLowerCase().includes(input.toLowerCase()),
-          )
-          .map((product) => {
-            const price = product.prices?.[0];
-            const priceStr =
-              price && 'priceAmount' in price && price.priceAmount
-                ? `$${(price.priceAmount / 100).toFixed(2)}/${price.recurringInterval || 'month'}`
-                : 'No price';
-            return {
-              name: `${product.name} (${priceStr})`,
-              value: product.id,
-            };
-          });
-      },
-    },
-    {
-      type: 'autocomplete',
-      name: 'organizationId',
-      message: 'Select organization:',
-      source: (answersSoFar: any, input = '') => {
-        return organizations
-          .filter(
-            (org) =>
-              org.name.toLowerCase().includes(input.toLowerCase()) ||
-              org.id.toLowerCase().includes(input.toLowerCase()),
-          )
-          .map((org) => ({
-            name: `${org.name} (${org.id})`,
-            value: org.id,
-          }));
-      },
-    },
-  ]);
-
-  return {
-    ...polarCredentials,
-    ...restOfAnswers,
-  };
-}
-
-async function main() {
-  console.log('Assigning existing product to organization...');
-  const input = await promptForInput();
-
-  const polar = new Polar({
-    accessToken: input.polarApiKey,
-    server: input.isProduction ? 'production' : 'sandbox',
-  });
-
-  const organization = await db.organization.findUniqueOrThrow({
-    where: {
-      id: input.organizationId,
-    },
-    select: {
-      id: true,
-      name: true,
-      createdBy: {
-        select: {
-          id: true,
-          email: true,
-          firstName: true,
-          lastName: true,
-        },
-      },
-      projects: {
-        select: {
-          id: true,
-        },
-      },
-    },
-  });
-
-  if (!organization.createdBy) {
-    throw new Error(
-      `Organization ${organization.name} does not have a creator. Cannot proceed.`,
-    );
-  }
-
-  const user = organization.createdBy;
-
-  // Fetch product details for review
-  const product = await polar.products.get({ id: input.productId });
-  const price = product.prices?.[0];
-  const priceStr =
-    price && 'priceAmount' in price && price.priceAmount
-      ? `$${(price.priceAmount / 100).toFixed(2)}/${price.recurringInterval || 'month'}`
-      : 'No price';
-
-  console.log('\nReview the following settings:');
-  console.table({
-    product: product.name,
-    price: priceStr,
-    organization: organization.name,
-    email: user.email,
-    name:
-      [user.firstName, user.lastName].filter(Boolean).join(' ') || 'No name',
-  });
-
-  const { confirmed } = await inquirer.prompt([
-    {
-      type: 'confirm',
-      name: 'confirmed',
-      message: 'Do you want to proceed?',
-      default: false,
-    },
-  ]);
-
-  if (!confirmed) {
-    console.log('Operation canceled');
-    return;
-  }
-
-  const checkoutLink = await polar.checkoutLinks.create({
-    paymentProcessor: 'stripe',
-    productId: input.productId,
-    allowDiscountCodes: false,
-    metadata: {
-      organizationId: organization.id,
-      userId: user.id,
-    },
-    successUrl: getSuccessUrl(
-      input.isProduction
-        ? 'https://dashboard.openpanel.dev'
-        : 'http://localhost:3000',
-      organization.id,
-    ),
-  });
-
-  console.log('\nCheckout link created:');
-  console.table(checkoutLink);
-  console.log('\nProduct assigned successfully!');
-}
-
-main()
-  .catch(console.error)
-  .finally(() => db.$disconnect());

packages/queue/src/queues.ts

@@ -6,7 +6,7 @@ import type {
 } from '@openpanel/db';
 import { createLogger } from '@openpanel/logger';
 import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
-import { Queue, QueueEvents } from 'bullmq';
+import { Queue } from 'bullmq';
 import { Queue as GroupQueue } from 'groupmq';
 import type { ITrackPayload } from '../../validation';
 
@@ -66,6 +66,10 @@ export interface EventsQueuePayloadIncomingEvent {
     headers: Record<string, string | undefined>;
     deviceId: string;
     sessionId: string;
+    session?: Pick<
+      IServiceCreateEventPayload,
+      'referrer' | 'referrerName' | 'referrerType'
+    >;
   };
 }
 export interface EventsQueuePayloadCreateEvent {
@@ -206,9 +210,6 @@ export const sessionsQueue = new Queue<SessionsQueuePayload>(
     },
   }
 );
-export const sessionsQueueEvents = new QueueEvents(getQueueName('sessions'), {
-  connection: getRedisQueue(),
-});
 
 export const cronQueue = new Queue<CronQueuePayload>(getQueueName('cron'), {
   connection: getRedisQueue(),

packages/redis/cachable.ts

@@ -1,7 +1,7 @@
 import { LRUCache } from 'lru-cache';
 import { getRedisCache } from './redis';
 
-export const deleteCache = async (key: string) => {
+export const deleteCache = (key: string) => {
   return getRedisCache().del(key);
 };
 
@@ -15,7 +15,7 @@ export async function getCache<T>(
   key: string,
   expireInSec: number,
   fn: () => Promise<T>,
-  useLruCache?: boolean,
+  useLruCache?: boolean
 ): Promise<T> {
   // L1 Cache: Check global LRU cache first (in-memory, instant)
   if (useLruCache) {
@@ -28,15 +28,7 @@ export async function getCache<T>(
   // L2 Cache: Check Redis cache (shared across instances)
   const hit = await getRedisCache().get(key);
   if (hit) {
-    const parsed = JSON.parse(hit, (_, value) => {
+    const parsed = parseCache(hit);
-      if (
-        typeof value === 'string' &&
-        /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
-      ) {
-        return new Date(value);
-      }
-      return value;
-    });
 
     // Store in LRU cache for next time
     if (useLruCache) {
@@ -81,12 +73,24 @@ export function getGlobalLruCacheStats() {
 }
 
 function stringify(obj: unknown): string {
-  if (obj === null) return 'null';
+  if (obj === null) {
-  if (obj === undefined) return 'undefined';
+    return 'null';
-  if (typeof obj === 'boolean') return obj ? 'true' : 'false';
+  }
-  if (typeof obj === 'number') return String(obj);
+  if (obj === undefined) {
-  if (typeof obj === 'string') return obj;
+    return 'undefined';
-  if (typeof obj === 'function') return obj.toString();
+  }
+  if (typeof obj === 'boolean') {
+    return obj ? 'true' : 'false';
+  }
+  if (typeof obj === 'number') {
+    return String(obj);
+  }
+  if (typeof obj === 'string') {
+    return obj;
+  }
+  if (typeof obj === 'function') {
+    return obj.toString();
+  }
 
   if (Array.isArray(obj)) {
     return `[${obj.map(stringify).join(',')}]`;
@@ -128,17 +132,29 @@ function hasResult(result: unknown): boolean {
   return true;
 }
 
-export interface CacheableLruOptions {
+const DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/;
-  /** TTL in seconds for LRU cache */
+const parseCache = (cached: string) => {
-  ttl: number;
+  try {
-  /** Maximum number of entries in LRU cache */
+    return JSON.parse(cached, (_, value) => {
-  maxSize?: number;
+      if (typeof value === 'string' && DATE_REGEX.test(value)) {
-}
+        return new Date(value);
+      }
+      return value;
+    });
+  } catch (error) {
+    console.error('Failed to parse cache', error);
+    return null;
+  }
+};
+
+// L1 cache: short TTL to offload Redis; clear() invalidates Redis, other nodes may serve stale from LRU for up to this long
+const CACHEABLE_LRU_TTL_MS = 60 * 1000; // 60 seconds
+const CACHEABLE_LRU_MAX = 1000;
 
 // Overload 1: cacheable(fn, expireInSec)
 export function cacheable<T extends (...args: any) => any>(
   fn: T,
-  expireInSec: number,
+  expireInSec: number
 ): T & {
   getKey: (...args: Parameters<T>) => string;
   clear: (...args: Parameters<T>) => Promise<number>;
@@ -151,7 +167,7 @@ export function cacheable<T extends (...args: any) => any>(
 export function cacheable<T extends (...args: any) => any>(
   name: string,
   fn: T,
-  expireInSec: number,
+  expireInSec: number
 ): T & {
   getKey: (...args: Parameters<T>) => string;
   clear: (...args: Parameters<T>) => Promise<number>;
@@ -164,7 +180,7 @@ export function cacheable<T extends (...args: any) => any>(
 export function cacheable<T extends (...args: any) => any>(
   fnOrName: T | string,
   fnOrExpireInSec: number | T,
-  _expireInSec?: number,
+  _expireInSec?: number
 ) {
   const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
   const fn =
@@ -195,184 +211,67 @@ export function cacheable<T extends (...args: any) => any>(
 
   const cachePrefix = `cachable:${name}`;
   const getKey = (...args: Parameters<T>) =>
-    `${cachePrefix}:${stringify(args)}`;
+    `${cachePrefix}:${stringify(args)}`.replaceAll(/\s/g, '');
 
-  // Redis-only mode: asynchronous implementation
+  const lruCache = new LRUCache<string, any>({
+    max: CACHEABLE_LRU_MAX,
+    ttl: CACHEABLE_LRU_TTL_MS,
+  });
+
+  // L1 LRU (60s) + L2 Redis. clear() deletes Redis + local LRU; other nodes may serve stale from LRU for up to 60s.
   const cachedFn = async (
     ...args: Parameters<T>
   ): Promise<Awaited<ReturnType<T>>> => {
     const key = getKey(...args);
 
-    // Check Redis cache (shared across instances)
+    // L1: in-memory LRU first (offloads Redis on hot keys)
+    const lruHit = lruCache.get(key);
+    if (lruHit !== undefined && hasResult(lruHit)) {
+      return lruHit as Awaited<ReturnType<T>>;
+    }
+
+    // L2: Redis (shared across instances)
     const cached = await getRedisCache().get(key);
     if (cached) {
-      try {
+      const parsed = parseCache(cached);
-        const parsed = JSON.parse(cached, (_, value) => {
+      if (hasResult(parsed)) {
-          if (
+        lruCache.set(key, parsed);
-            typeof value === 'string' &&
+        return parsed;
-            /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
-          ) {
-            return new Date(value);
-          }
-          return value;
-        });
-        if (hasResult(parsed)) {
-          return parsed;
-        }
-      } catch (e) {
-        console.error('Failed to parse cache', e);
       }
     }
 
-    // Cache miss: Execute function
+    // Cache miss: execute function
     const result = await fn(...(args as any));
 
     if (hasResult(result)) {
-      // Don't await Redis write - fire and forget for better performance
+      lruCache.set(key, result);
       getRedisCache()
         .setex(key, expireInSec, JSON.stringify(result))
-        .catch(() => {});
+        .catch(() => {
+          // ignore error
+        });
     }
 
     return result;
   };
 
-  cachedFn.getKey = getKey;
-  cachedFn.clear = async (...args: Parameters<T>) => {
-    const key = getKey(...args);
-    return getRedisCache().del(key);
-  };
-  cachedFn.set =
-    (...args: Parameters<T>) =>
-    async (payload: Awaited<ReturnType<T>>) => {
-      const key = getKey(...args);
-      return getRedisCache()
-        .setex(key, expireInSec, JSON.stringify(payload))
-        .catch(() => {});
-    };
-
-  return cachedFn;
-}
-
-// Overload 1: cacheableLru(fn, options)
-export function cacheableLru<T extends (...args: any) => any>(
-  fn: T,
-  options: CacheableLruOptions,
-): T & {
-  getKey: (...args: Parameters<T>) => string;
-  clear: (...args: Parameters<T>) => boolean;
-  set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
-};
-
-// Overload 2: cacheableLru(name, fn, options)
-export function cacheableLru<T extends (...args: any) => any>(
-  name: string,
-  fn: T,
-  options: CacheableLruOptions,
-): T & {
-  getKey: (...args: Parameters<T>) => string;
-  clear: (...args: Parameters<T>) => boolean;
-  set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
-};
-
-// Implementation for cacheableLru (LRU-only - synchronous)
-export function cacheableLru<T extends (...args: any) => any>(
-  fnOrName: T | string,
-  fnOrOptions: T | CacheableLruOptions,
-  _options?: CacheableLruOptions,
-) {
-  const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
-  const fn =
-    typeof fnOrName === 'function'
-      ? fnOrName
-      : typeof fnOrOptions === 'function'
-        ? fnOrOptions
-        : null;
-
-  let options: CacheableLruOptions;
-
-  // Parse parameters based on function signature
-  if (typeof fnOrName === 'function') {
-    // Overload 1: cacheableLru(fn, options)
-    options =
-      typeof fnOrOptions === 'object' && fnOrOptions !== null
-        ? fnOrOptions
-        : ({} as CacheableLruOptions);
-  } else {
-    // Overload 2: cacheableLru(name, fn, options)
-    options =
-      typeof _options === 'object' && _options !== null
-        ? _options
-        : ({} as CacheableLruOptions);
-  }
-
-  if (typeof fn !== 'function') {
-    throw new Error('fn is not a function');
-  }
-
-  if (typeof options.ttl !== 'number') {
-    throw new Error('options.ttl is required and must be a number');
-  }
-
-  const cachePrefix = `cachable:${name}`;
-  const getKey = (...args: Parameters<T>) =>
-    `${cachePrefix}:${stringify(args)}`;
-
-  const maxSize = options.maxSize ?? 1000;
-  const ttl = options.ttl;
-
-  // Create function-specific LRU cache
-  const functionLruCache = new LRUCache<string, any>({
-    max: maxSize,
-    ttl: ttl * 1000, // Convert seconds to milliseconds for LRU
-  });
-
-  // LRU-only mode: synchronous implementation (or returns promise if fn is async)
-  const cachedFn = ((...args: Parameters<T>): ReturnType<T> => {
-    const key = getKey(...args);
-
-    // Check LRU cache
-    const lruHit = functionLruCache.get(key);
-    if (lruHit !== undefined && hasResult(lruHit)) {
-      return lruHit as ReturnType<T>;
-    }
-
-    // Cache miss: Execute function
-    const result = fn(...(args as any)) as ReturnType<T>;
-
-    // If result is a Promise, handle it asynchronously but cache the resolved value
-    if (result && typeof (result as any).then === 'function') {
-      return (result as Promise<any>).then((resolved: any) => {
-        if (hasResult(resolved)) {
-          functionLruCache.set(key, resolved);
-        }
-        return resolved;
-      }) as ReturnType<T>;
-    }
-
-    // Synchronous result: cache and return
-    if (hasResult(result)) {
-      functionLruCache.set(key, result);
-    }
-
-    return result;
-  }) as T & {
-    getKey: (...args: Parameters<T>) => string;
-    clear: (...args: Parameters<T>) => boolean;
-    set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
-  };
-
   cachedFn.getKey = getKey;
   cachedFn.clear = (...args: Parameters<T>) => {
     const key = getKey(...args);
-    return functionLruCache.delete(key);
+    lruCache.delete(key);
+    return getRedisCache().del(key);
   };
   cachedFn.set =
     (...args: Parameters<T>) =>
-    (payload: ReturnType<T>) => {
+    (payload: Awaited<ReturnType<T>>) => {
       const key = getKey(...args);
       if (hasResult(payload)) {
-        functionLruCache.set(key, payload);
+        lruCache.set(key, payload);
+        return getRedisCache()
+          .setex(key, expireInSec, JSON.stringify(payload))
+          .catch(() => {
+            // ignore error
+          });
       }
     };
 

packages/redis/publisher.ts

@@ -10,8 +10,7 @@ export type IPublishChannels = {
     };
   };
   events: {
-    received: IServiceEvent;
+    batch: { projectId: string; count: number };
-    saved: IServiceEvent;
   };
   notification: {
     created: Prisma.NotificationUncheckedCreateInput;

packages/sdks/react-native/index.ts

@@ -1,13 +1,13 @@
+import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
+import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
 import * as Application from 'expo-application';
 import Constants from 'expo-constants';
 import { AppState, Platform } from 'react-native';
 
-import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
-import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
-
 export * from '@openpanel/sdk';
 
 export class OpenPanel extends OpenPanelBase {
+  private lastPath = '';
   constructor(public options: OpenPanelOptions) {
     super({
       ...options,
@@ -37,7 +37,12 @@ export class OpenPanel extends OpenPanelBase {
     });
   }
 
-  public screenView(route: string, properties?: TrackProperties): void {
+  track(name: string, properties?: TrackProperties) {
+    return super.track(name, { ...properties, __path: this.lastPath });
+  }
+
+  screenView(route: string, properties?: TrackProperties): void {
+    this.lastPath = route;
     super.track('screen_view', {
       ...properties,
       __path: route,

packages/sdks/web/src/index.ts

@@ -58,7 +58,7 @@ export type OpenPanelOptions = OpenPanelBaseOptions & {
 
 function toCamelCase(str: string) {
   return str.replace(/([-_][a-z])/gi, ($1) =>
-    $1.toUpperCase().replace('-', '').replace('_', ''),
+    $1.toUpperCase().replace('-', '').replace('_', '')
   );
 }
 
@@ -114,7 +114,9 @@ export class OpenPanel extends OpenPanelBase {
         const sampled = Math.random() < sampleRate;
         if (sampled) {
           this.loadReplayModule().then((mod) => {
-            if (!mod) return;
+            if (!mod) {
+              return;
+            }
             mod.startReplayRecorder(this.options.sessionReplay!, (chunk) => {
               // Replay chunks go through send() and are queued when disabled or waitForProfile
               // until ready() is called (base SDK also queues replay until sessionId is set).
@@ -153,7 +155,10 @@ export class OpenPanel extends OpenPanelBase {
       // dead-code-eliminated in the library build.
       if (typeof __OPENPANEL_REPLAY_URL__ !== 'undefined') {
         const scriptEl = _replayScriptRef;
-        const url = this.options.sessionReplay?.scriptUrl || scriptEl?.src?.replace('.js', '-replay.js') || 'https://openpanel.dev/op1-replay.js';
+        const url =
+          this.options.sessionReplay?.scriptUrl ||
+          scriptEl?.src?.replace('.js', '-replay.js') ||
+          'https://openpanel.dev/op1-replay.js';
 
         // Already loaded (e.g. user included the script manually)
         if ((window as any).__openpanel_replay) {
@@ -287,11 +292,15 @@ export class OpenPanel extends OpenPanelBase {
     });
   }
 
+  track(name: string, properties?: TrackProperties) {
+    return super.track(name, { ...properties, __path: this.lastPath });
+  }
+
   screenView(properties?: TrackProperties): void;
   screenView(path: string, properties?: TrackProperties): void;
   screenView(
     pathOrProperties?: string | TrackProperties,
-    propertiesOrUndefined?: TrackProperties,
+    propertiesOrUndefined?: TrackProperties
   ): void {
     if (this.isServer()) {
       return;
@@ -322,7 +331,7 @@ export class OpenPanel extends OpenPanelBase {
 
   async flushRevenue() {
     const promises = this.pendingRevenues.map((pending) =>
-      super.revenue(pending.amount, pending.properties),
+      super.revenue(pending.amount, pending.properties)
     );
     await Promise.all(promises);
     this.clearRevenue();
@@ -343,7 +352,7 @@ export class OpenPanel extends OpenPanelBase {
       try {
         sessionStorage.setItem(
           'openpanel-pending-revenues',
-          JSON.stringify(this.pendingRevenues),
+          JSON.stringify(this.pendingRevenues)
         );
       } catch {}
     }

packages/trpc/src/routers/chart.ts

@@ -1,11 +1,7 @@
-import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
+import { round } from '@openpanel/common';
-import sqlstring from 'sqlstring';
-import { z } from 'zod';
-
 import {
-  type IClickhouseProfile,
+  AggregateChartEngine,
-  type IServiceProfile,
+  ChartEngine,
-  TABLE_NAMES,
   ch,
   chQuery,
   clix,
@@ -21,8 +17,11 @@ import {
   getReportById,
   getSelectPropertyKey,
   getSettingsForProject,
+  type IClickhouseProfile,
+  type IServiceProfile,
   onlyReportEvents,
   sankeyService,
+  TABLE_NAMES,
   validateShareAccess,
 } from '@openpanel/db';
 import {
@@ -33,15 +32,15 @@ import {
   zReportInput,
   zTimeInterval,
 } from '@openpanel/validation';
-
-import { round } from '@openpanel/common';
-import { AggregateChartEngine, ChartEngine } from '@openpanel/db';
 import {
   differenceInDays,
   differenceInMonths,
   differenceInWeeks,
   formatISO,
 } from 'date-fns';
+import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
+import sqlstring from 'sqlstring';
+import { z } from 'zod';
 import { getProjectAccess } from '../access';
 import { TRPCAccessError } from '../errors';
 import {
@@ -83,7 +82,7 @@ const chartProcedure = publicProcedure.use(
           session: ctx.session?.userId
             ? { userId: ctx.session.userId }
             : undefined,
-        },
+        }
       );
       if (!shareValidation.isValid) {
         throw TRPCAccessError('You do not have access to this share');
@@ -119,7 +118,7 @@ const chartProcedure = publicProcedure.use(
         report: null,
       },
     });
-  },
+  }
 );
 
 export const chartRouter = createTRPCRouter({
@@ -128,7 +127,7 @@ export const chartRouter = createTRPCRouter({
     .input(
       z.object({
         projectId: z.string(),
-      }),
+      })
     )
     .query(async ({ input: { projectId } }) => {
       const { timezone } = await getSettingsForProject(projectId);
@@ -151,7 +150,7 @@ export const chartRouter = createTRPCRouter({
         TO toStartOfDay(now()) 
         STEP INTERVAL 1 day
         SETTINGS session_timezone = '${timezone}'
-      `,
+      `
       );
 
       const metricsPromise = clix(ch, timezone)
@@ -185,7 +184,7 @@ export const chartRouter = createTRPCRouter({
           ? Math.round(
               ((metrics.months_3 - metrics.months_3_prev) /
                 metrics.months_3_prev) *
-                100,
+                100
             )
           : null;
 
@@ -209,12 +208,12 @@ export const chartRouter = createTRPCRouter({
     .input(
       z.object({
         projectId: z.string(),
-      }),
+      })
     )
     .query(async ({ input: { projectId } }) => {
       const [events, meta] = await Promise.all([
         chQuery<{ name: string; count: number }>(
-          `SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`,
+          `SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`
         ),
         getEventMetasCached(projectId),
       ]);
@@ -238,7 +237,7 @@ export const chartRouter = createTRPCRouter({
       z.object({
         event: z.string().optional(),
         projectId: z.string(),
-      }),
+      })
     )
     .query(async ({ input: { projectId, event } }) => {
       const profiles = await clix(ch, 'UTC')
@@ -252,8 +251,8 @@ export const chartRouter = createTRPCRouter({
       const profileProperties = [
         ...new Set(
           profiles.flatMap((p) =>
-            Object.keys(p.properties).map((k) => `profile.properties.${k}`),
+            Object.keys(p.properties).map((k) => `profile.properties.${k}`)
-          ),
+          )
         ),
       ];
 
@@ -283,7 +282,6 @@ export const chartRouter = createTRPCRouter({
       });
 
       const fixedProperties = [
-        'duration',
         'revenue',
         'has_profile',
         'path',
@@ -316,7 +314,7 @@ export const chartRouter = createTRPCRouter({
 
       return pipe(
         sort<string>((a, b) => a.length - b.length),
-        uniq,
+        uniq
       )(properties);
     }),
 
@@ -326,9 +324,9 @@ export const chartRouter = createTRPCRouter({
         event: z.string(),
         property: z.string(),
         projectId: z.string(),
-      }),
+      })
     )
-    .query(async ({ input: { event, property, projectId, ...input } }) => {
+    .query(async ({ input: { event, property, projectId } }) => {
       if (property === 'has_profile') {
         return {
           values: ['true', 'false'],
@@ -378,7 +376,7 @@ export const chartRouter = createTRPCRouter({
               .from(TABLE_NAMES.profiles)
               .where('project_id', '=', projectId),
             'profile.id = profile_id',
-            'profile',
+            'profile'
           );
         }
 
@@ -389,8 +387,8 @@ export const chartRouter = createTRPCRouter({
             (data: typeof events) => map(prop('values'), data),
             flatten,
             uniq,
-            sort((a, b) => a.length - b.length),
+            sort((a, b) => a.length - b.length)
-          )(events),
+          )(events)
         );
       }
 
@@ -406,8 +404,8 @@ export const chartRouter = createTRPCRouter({
         z.object({
           shareId: z.string().optional(),
           id: z.string().optional(),
-        }),
+        })
-      ),
+      )
     )
     .query(async ({ input, ctx }) => {
       const chartInput = ctx.report
@@ -448,8 +446,8 @@ export const chartRouter = createTRPCRouter({
         z.object({
           shareId: z.string().optional(),
           id: z.string().optional(),
-        }),
+        })
-      ),
+      )
     )
     .query(async ({ input, ctx }) => {
       const chartInput = ctx.report
@@ -536,12 +534,10 @@ export const chartRouter = createTRPCRouter({
         z.object({
           shareId: z.string().optional(),
           id: z.string().optional(),
-        }),
+        })
-      ),
+      )
     )
-    .query(async ({ input, ctx }) => {
+    .query(({ input, ctx }) => {
-      console.log('input', input);
-
       const chartInput = ctx.report
         ? {
             ...ctx.report,
@@ -562,10 +558,10 @@ export const chartRouter = createTRPCRouter({
         z.object({
           shareId: z.string().optional(),
           id: z.string().optional(),
-        }),
+        })
-      ),
+      )
     )
-    .query(async ({ input, ctx }) => {
+    .query(({ input, ctx }) => {
       const chartInput = ctx.report
         ? {
             ...ctx.report,
@@ -593,7 +589,7 @@ export const chartRouter = createTRPCRouter({
         range: zRange,
         shareId: z.string().optional(),
         id: z.string().optional(),
-      }),
+      })
     )
     .query(async ({ input, ctx }) => {
       const projectId = ctx.report?.projectId ?? input.projectId;
@@ -647,7 +643,7 @@ export const chartRouter = createTRPCRouter({
           startDate,
           endDate,
         },
-        timezone,
+        timezone
       );
       const diffInterval = {
         minute: () => differenceInDays(dates.endDate, dates.startDate),
@@ -677,14 +673,14 @@ export const chartRouter = createTRPCRouter({
       const usersSelect = range(0, diffInterval + 1)
         .map(
           (index) =>
-            `groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`,
+            `groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`
         )
         .join(',\n');
 
       const countsSelect = range(0, diffInterval + 1)
         .map(
           (index) =>
-            `length(interval_${index}_users) AS interval_${index}_user_count`,
+            `length(interval_${index}_users) AS interval_${index}_user_count`
         )
         .join(',\n');
 
@@ -769,12 +765,10 @@ export const chartRouter = createTRPCRouter({
         interval: zTimeInterval.default('day'),
         series: zChartSeries,
         breakdowns: z.record(z.string(), z.string()).optional(),
-      }),
+      })
     )
     .query(async ({ input }) => {
-      const { timezone } = await getSettingsForProject(input.projectId);
       const { projectId, date, series } = input;
-      const limit = 100;
       const serie = series[0];
 
       if (!serie) {
@@ -813,7 +807,7 @@ export const chartRouter = createTRPCRouter({
       if (profileFields.length > 0) {
         // Extract top-level field names and select only what's needed
         const fieldsToSelect = uniq(
-          profileFields.map((f) => f.split('.')[0]),
+          profileFields.map((f) => f.split('.')[0])
         ).join(', ');
         sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
       }
@@ -836,7 +830,7 @@ export const chartRouter = createTRPCRouter({
       // Fetch profile details in batches to avoid exceeding ClickHouse max_query_size
       const ids = profileIds.map((p) => p.profile_id).filter(Boolean);
       const BATCH_SIZE = 200;
-      const profiles = [];
+      const profiles: IServiceProfile[] = [];
       for (let i = 0; i < ids.length; i += BATCH_SIZE) {
         const batch = ids.slice(i, i + BATCH_SIZE);
         const batchProfiles = await getProfilesCached(batch, projectId);
@@ -859,13 +853,13 @@ export const chartRouter = createTRPCRouter({
           .optional()
           .default(false)
           .describe(
-            'If true, show users who dropped off at this step. If false, show users who completed at least this step.',
+            'If true, show users who dropped off at this step. If false, show users who completed at least this step.'
           ),
         funnelWindow: z.number().optional(),
         funnelGroup: z.string().optional(),
         breakdowns: z.array(z.object({ name: z.string() })).optional(),
         range: zRange,
-      }),
+      })
     )
     .query(async ({ input }) => {
       const { timezone } = await getSettingsForProject(input.projectId);
@@ -911,15 +905,15 @@ export const chartRouter = createTRPCRouter({
 
       // Check for profile filters and add profile join if needed
       const profileFilters = funnelService.getProfileFilters(
-        eventSeries as IChartEvent[],
+        eventSeries as IChartEvent[]
       );
       if (profileFilters.length > 0) {
         const fieldsToSelect = uniq(
-          profileFilters.map((f) => f.split('.')[0]),
+          profileFilters.map((f) => f.split('.')[0])
         ).join(', ');
         funnelCte.leftJoin(
           `(SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile`,
-          'profile.id = events.profile_id',
+          'profile.id = events.profile_id'
         );
       }
 
@@ -934,7 +928,7 @@ export const chartRouter = createTRPCRouter({
         // `max(level) AS level` alias (ILLEGAL_AGGREGATION error).
         query.with(
           'funnel',
-          'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id',
+          'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id'
         );
       } else {
         // For session grouping: filter out level = 0 inside the CTE
@@ -969,7 +963,7 @@ export const chartRouter = createTRPCRouter({
       // when there are many profile IDs to pass in the IN(...) clause
       const ids = profileIdsResult.map((p) => p.profile_id).filter(Boolean);
       const BATCH_SIZE = 500;
-      const profiles = [];
+      const profiles: IServiceProfile[] = [];
       for (let i = 0; i < ids.length; i += BATCH_SIZE) {
         const batch = ids.slice(i, i + BATCH_SIZE);
         const batchProfiles = await getProfilesCached(batch, projectId);
@@ -986,7 +980,7 @@ function processCohortData(
     total_first_event_count: number;
     [key: string]: any;
   }>,
-  diffInterval: number,
+  diffInterval: number
 ) {
   if (data.length === 0) {
     return [];
@@ -995,13 +989,13 @@ function processCohortData(
   const processed = data.map((row) => {
     const sum = row.total_first_event_count;
     const values = range(0, diffInterval + 1).map(
-      (index) => (row[`interval_${index}_user_count`] || 0) as number,
+      (index) => (row[`interval_${index}_user_count`] || 0) as number
     );
 
     return {
       cohort_interval: row.cohort_interval,
       sum,
-      values: values,
+      values,
       percentages: values.map((value) => (sum > 0 ? round(value / sum, 2) : 0)),
     };
   });
@@ -1041,10 +1035,10 @@ function processCohortData(
     cohort_interval: 'Weighted Average',
     sum: round(averageData.totalSum / processed.length, 0),
     percentages: averageData.percentages.map(({ sum, weightedSum }) =>
-      sum > 0 ? round(weightedSum / sum, 2) : 0,
+      sum > 0 ? round(weightedSum / sum, 2) : 0
     ),
     values: averageData.values.map(({ sum, weightedSum }) =>
-      sum > 0 ? round(weightedSum / sum, 0) : 0,
+      sum > 0 ? round(weightedSum / sum, 0) : 0
     ),
   };
 

packages/trpc/src/routers/project.ts

@@ -96,9 +96,7 @@ export const projectRouter = createTRPCRouter({
       });
       await Promise.all([
         getProjectByIdCached.clear(input.id),
-        res.clients.map((client) => {
+        ...res.clients.map((client) => getClientByIdCached.clear(client.id)),
-          getClientByIdCached.clear(client.id);
-        }),
       ]);
       return res;
     }),

packages/trpc/src/routers/realtime.ts

@@ -1,18 +1,15 @@
-import { z } from 'zod';
-
 import {
-  type EventMeta,
-  TABLE_NAMES,
   ch,
   chQuery,
   clix,
-  db,
   formatClickhouseDate,
-  getEventList,
+  type IClickhouseEvent,
+  TABLE_NAMES,
+  transformEvent,
 } from '@openpanel/db';
-
 import { subMinutes } from 'date-fns';
 import sqlstring from 'sqlstring';
+import { z } from 'zod';
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 export const realtimeRouter = createTRPCRouter({
@@ -25,7 +22,7 @@ export const realtimeRouter = createTRPCRouter({
         long: number;
         lat: number;
       }>(
-        `SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`,
+        `SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
       );
 
       return res;
@@ -33,25 +30,18 @@ export const realtimeRouter = createTRPCRouter({
   activeSessions: protectedProcedure
     .input(z.object({ projectId: z.string() }))
     .query(async ({ input }) => {
-      return getEventList({
+      const rows = await chQuery<IClickhouseEvent>(
-        projectId: input.projectId,
+        `SELECT
-        take: 30,
+          name, session_id, created_at, path, origin, referrer, referrer_name,
-        select: {
+          country, city, region, os, os_version, browser, browser_version,
-          name: true,
+          device
-          path: true,
+        FROM ${TABLE_NAMES.events}
-          origin: true,
+        WHERE project_id = ${sqlstring.escape(input.projectId)}
-          referrer: true,
+          AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}'
-          referrerName: true,
+        ORDER BY created_at DESC
-          referrerType: true,
+        LIMIT 50`
-          country: true,
+      );
-          device: true,
+      return rows.map(transformEvent);
-          os: true,
-          browser: true,
-          createdAt: true,
-          profile: true,
-          meta: true,
-        },
-      });
     }),
   paths: protectedProcedure
     .input(z.object({ projectId: z.string() }))
@@ -76,7 +66,7 @@ export const realtimeRouter = createTRPCRouter({
         .where(
           'created_at',
           '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
         )
         .groupBy(['path', 'origin'])
         .orderBy('count', 'DESC')
@@ -106,7 +96,7 @@ export const realtimeRouter = createTRPCRouter({
         .where(
           'created_at',
           '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
         )
         .groupBy(['referrer_name'])
         .orderBy('count', 'DESC')
@@ -137,7 +127,7 @@ export const realtimeRouter = createTRPCRouter({
         .where(
           'created_at',
           '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
         )
         .groupBy(['country', 'city'])
         .orderBy('count', 'DESC')

pnpm-lock.yaml

@@ -16,8 +16,8 @@ catalogs:
       specifier: ^19.2.3
       version: 19.2.3
     groupmq:
-      specifier: 1.1.1-next.2
+      specifier: 2.0.0-next.1
-      version: 1.1.1-next.2
+      version: 2.0.0-next.1
     react:
       specifier: ^19.2.3
       version: 19.2.3
@@ -198,7 +198,7 @@ importers:
         version: 5.0.0
       groupmq:
         specifier: 'catalog:'
-        version: 1.1.1-next.2(ioredis@5.8.2)
+        version: 2.0.0-next.1(ioredis@5.8.2)
       jsonwebtoken:
         specifier: ^9.0.2
         version: 9.0.2
@@ -936,7 +936,7 @@ importers:
         version: 4.18.2
       groupmq:
         specifier: 'catalog:'
-        version: 1.1.1-next.2(ioredis@5.8.2)
+        version: 2.0.0-next.1(ioredis@5.8.2)
       prom-client:
         specifier: ^15.1.3
         version: 15.1.3
@@ -1419,7 +1419,7 @@ importers:
         version: 5.63.0
       groupmq:
         specifier: 'catalog:'
-        version: 1.1.1-next.2(ioredis@5.8.2)
+        version: 2.0.0-next.1(ioredis@5.8.2)
     devDependencies:
       '@openpanel/tsconfig':
         specifier: workspace:*
@@ -13157,11 +13157,11 @@ packages:
 
   glob@7.1.6:
     resolution: {integrity: sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==}
-    deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
+    deprecated: Glob versions prior to v9 are no longer supported
 
   glob@7.2.3:
     resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
-    deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
+    deprecated: Glob versions prior to v9 are no longer supported
 
   glob@9.3.5:
     resolution: {integrity: sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==}
@@ -13221,8 +13221,8 @@ packages:
     resolution: {integrity: sha512-5gghUc24tP9HRznNpV2+FIoq3xKkj5dTQqf4v0CpdPbFVwFkWoxOM+o+2OC9ZSvjEMTjfmG9QT+gcvggTwW1zw==}
     engines: {node: '>= 10.x'}
 
-  groupmq@1.1.1-next.2:
+  groupmq@2.0.0-next.1:
-    resolution: {integrity: sha512-5gH+P3NfSCjfCLcB2g2TAHCpmQz+rwrQkb+kAyrzB9puZuAHKQVYOUPWKVBRFjY7B9jPRGHrimDO6h9rWKGfMA==}
+    resolution: {integrity: sha512-xcpz29HeXXn0yP/sQTGPPNMLQAZCCrJg3x9kpOAFbtsXki5KVeBsY3mWNBt3Z+YCa9OxwkTFL6tOcrB67z127A==}
     engines: {node: '>=18'}
     peerDependencies:
       ioredis: '>=5'
@@ -34142,7 +34142,7 @@ snapshots:
 
   graphql@15.8.0: {}
 
-  groupmq@1.1.1-next.2(ioredis@5.8.2):
+  groupmq@2.0.0-next.1(ioredis@5.8.2):
     dependencies:
       cron-parser: 4.9.0
       ioredis: 5.8.2

pnpm-workspace.yaml

@@ -13,4 +13,4 @@ catalog:
   "@types/react-dom": ^19.2.3
   "@types/node": ^24.7.1
   typescript: ^5.9.3
-  groupmq: 1.1.1-next.2
+  groupmq: 2.0.0-next.1