feat: session replay

* wip

* wip

* wip

* wip

* final fixes

* comments

* fix

apps/worker/src/boot-cron.ts

@@ -63,6 +63,11 @@ export async function bootCron() {
       type: 'flushProfileBackfill',
       pattern: 1000 * 30,
     },
+    {
+      name: 'flush',
+      type: 'flushReplay',
+      pattern: 1000 * 10,
+    },
     {
       name: 'insightsDaily',
       type: 'insightsDaily',

apps/worker/src/jobs/cron.ts

@@ -1,6 +1,6 @@
 import type { Job } from 'bullmq';
 
-import { eventBuffer, profileBackfillBuffer, profileBuffer, sessionBuffer } from '@openpanel/db';
+import { eventBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
 import type { CronQueuePayload } from '@openpanel/queue';
 
 import { jobdeleteProjects } from './cron.delete-projects';
@@ -26,6 +26,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
     case 'flushProfileBackfill': {
       return await profileBackfillBuffer.tryFlush();
     }
+    case 'flushReplay': {
+      return await replayBuffer.tryFlush();
+    }
     case 'ping': {
       return await ping();
     }

apps/worker/src/jobs/events.incoming-event.ts

@@ -15,7 +15,6 @@ import {
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
 import * as R from 'ramda';
-import { v4 as uuid } from 'uuid';
 import { logger as baseLogger } from '@/utils/logger';
 import { createSessionEndJob, getSessionEnd } from '@/utils/session-handler';
 
@@ -53,10 +52,9 @@ async function createEventAndNotify(
   logger.info('Creating event', { event: payload });
   const [event] = await Promise.all([
     createEvent(payload),
-    checkNotificationRulesForEvent(payload).catch(() => {}),
+    checkNotificationRulesForEvent(payload).catch(() => null),
   ]);
 
-  console.log('Event created:', event);
   return event;
 }
 
@@ -87,6 +85,8 @@ export async function incomingEvent(
     projectId,
     currentDeviceId,
     previousDeviceId,
+    deviceId,
+    sessionId,
     uaInfo: _uaInfo,
   } = jobPayload;
   const properties = body.properties ?? {};
@@ -157,7 +157,6 @@ export async function incomingEvent(
         : undefined,
   } as const;
 
-  console.log('HERE?');
   // if timestamp is from the past we dont want to create a new session
   if (uaInfo.isServer || isTimestampFromThePast) {
     const session = profileId
@@ -167,8 +166,6 @@ export async function incomingEvent(
         })
       : null;
 
-    console.log('Server?');
-
     const payload = {
       ...baseEvent,
       deviceId: session?.device_id ?? '',
@@ -194,31 +191,31 @@ export async function incomingEvent(
 
     return createEventAndNotify(payload as IServiceEvent, logger, projectId);
   }
-  console.log('not?');
+
   const sessionEnd = await getSessionEnd({
     projectId,
     currentDeviceId,
     previousDeviceId,
+    deviceId,
     profileId,
   });
-  console.log('Server?');
-  const lastScreenView = sessionEnd
+  const activeSession = sessionEnd
     ? await sessionBuffer.getExistingSession({
         sessionId: sessionEnd.sessionId,
       })
     : null;
 
   const payload: IServiceCreateEventPayload = merge(baseEvent, {
-    deviceId: sessionEnd?.deviceId ?? currentDeviceId,
-    sessionId: sessionEnd?.sessionId ?? uuid(),
+    deviceId: sessionEnd?.deviceId ?? deviceId,
+    sessionId: sessionEnd?.sessionId ?? sessionId,
     referrer: sessionEnd?.referrer ?? baseEvent.referrer,
     referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,
     referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,
     // if the path is not set, use the last screen view path
-    path: baseEvent.path || lastScreenView?.exit_path || '',
-    origin: baseEvent.origin || lastScreenView?.exit_origin || '',
+    path: baseEvent.path || activeSession?.exit_path || '',
+    origin: baseEvent.origin || activeSession?.exit_origin || '',
   } as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;
-  console.log('SessionEnd?', sessionEnd);
+
   if (!sessionEnd) {
     logger.info('Creating session start event', { event: payload });
     await createEventAndNotify(

apps/worker/src/jobs/events.incoming-events.test.ts

@@ -32,6 +32,8 @@ const SESSION_TIMEOUT = 30 * 60 * 1000;
 const projectId = 'test-project';
 const currentDeviceId = 'device-123';
 const previousDeviceId = 'device-456';
+// Valid UUID used when creating a new session in tests
+const newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';
 const geo = {
   country: 'US',
   city: 'New York',
@@ -67,7 +69,7 @@ describe('incomingEvent', () => {
     vi.clearAllMocks();
   });
 
-  it.only('should create a session start and an event', async () => {
+  it('should create a session start and an event', async () => {
     const spySessionsQueueAdd = vi.spyOn(sessionsQueue, 'add');
     const timestamp = new Date();
     // Mock job data
@@ -90,12 +92,15 @@ describe('incomingEvent', () => {
       projectId,
       currentDeviceId,
       previousDeviceId,
+      deviceId: currentDeviceId,
+      sessionId: newSessionId,
     };
     const event = {
       name: 'test_event',
       deviceId: currentDeviceId,
       profileId: '',
       sessionId: expect.stringMatching(
+        // biome-ignore lint/performance/useTopLevelRegex: test
         /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
       ),
       projectId,
@@ -182,6 +187,8 @@ describe('incomingEvent', () => {
       projectId,
       currentDeviceId,
       previousDeviceId,
+      deviceId: currentDeviceId,
+      sessionId: 'session-123',
     };
 
     const changeDelay = vi.fn();
@@ -263,6 +270,8 @@ describe('incomingEvent', () => {
       projectId,
       currentDeviceId: '',
       previousDeviceId: '',
+      deviceId: '',
+      sessionId: '',
       uaInfo: uaInfoServer,
     };
 
@@ -367,6 +376,8 @@ describe('incomingEvent', () => {
       projectId,
       currentDeviceId: '',
       previousDeviceId: '',
+      deviceId: '',
+      sessionId: '',
       uaInfo: uaInfoServer,
     };
 

apps/worker/src/metrics.ts

@@ -4,6 +4,7 @@ import {
   botBuffer,
   eventBuffer,
   profileBuffer,
+  replayBuffer,
   sessionBuffer,
 } from '@openpanel/db';
 import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
@@ -124,3 +125,14 @@ register.registerMetric(
     },
   }),
 );
+
+register.registerMetric(
+  new client.Gauge({
+    name: `buffer_${replayBuffer.name}_count`,
+    help: 'Number of unprocessed replay chunks',
+    async collect() {
+      const metric = await replayBuffer.getBufferSize();
+      this.set(metric);
+    },
+  }),
+);

apps/worker/src/utils/session-handler.ts

@@ -39,17 +39,20 @@ export async function getSessionEnd({
   projectId,
   currentDeviceId,
   previousDeviceId,
+  deviceId,
   profileId,
 }: {
   projectId: string;
   currentDeviceId: string;
   previousDeviceId: string;
+  deviceId: string;
   profileId: string;
 }) {
   const sessionEnd = await getSessionEndJob({
     projectId,
     currentDeviceId,
     previousDeviceId,
+    deviceId,
   });
 
   if (sessionEnd) {
@@ -81,6 +84,7 @@ export async function getSessionEndJob(args: {
   projectId: string;
   currentDeviceId: string;
   previousDeviceId: string;
+  deviceId: string;
   retryCount?: number;
 }): Promise<{
   deviceId: string;
@@ -130,20 +134,31 @@ export async function getSessionEndJob(args: {
     return null;
   }
 
-  // Check current device job
-  const currentJob = await sessionsQueue.getJob(
-    getSessionEndJobId(args.projectId, args.currentDeviceId),
-  );
-  if (currentJob) {
-    return await handleJobStates(currentJob, args.currentDeviceId);
+  // TODO: Remove this when migrated to deviceId
+  if (args.currentDeviceId && args.previousDeviceId) {
+    // Check current device job
+    const currentJob = await sessionsQueue.getJob(
+      getSessionEndJobId(args.projectId, args.currentDeviceId),
+    );
+    if (currentJob) {
+      return await handleJobStates(currentJob, args.currentDeviceId);
+    }
+
+    // Check previous device job
+    const previousJob = await sessionsQueue.getJob(
+      getSessionEndJobId(args.projectId, args.previousDeviceId),
+    );
+    if (previousJob) {
+      return await handleJobStates(previousJob, args.previousDeviceId);
+    }
   }
 
-  // Check previous device job
-  const previousJob = await sessionsQueue.getJob(
-    getSessionEndJobId(args.projectId, args.previousDeviceId),
+  // Check current device job
+  const currentJob = await sessionsQueue.getJob(
+    getSessionEndJobId(args.projectId, args.deviceId),
   );
-  if (previousJob) {
-    return await handleJobStates(previousJob, args.previousDeviceId);
+  if (currentJob) {
+    return await handleJobStates(currentJob, args.deviceId);
   }
 
   // Create session