improve(queue): how we handle incoming events and session ends

apps/worker/src/jobs/events.create-session-end.ts

@@ -1,5 +1,4 @@
 import type { Job } from 'bullmq';
-import { last } from 'ramda';
 
 import { logger as baseLogger } from '@/utils/logger';
 import { getTime } from '@openpanel/common';
@@ -9,61 +8,56 @@ import {
   checkNotificationRulesForSessionEnd,
   createEvent,
   eventBuffer,
+  formatClickhouseDate,
   getEvents,
 } from '@openpanel/db';
-import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';
 
-async function getCompleteSession({
+// Grabs session_start and screen_views + the last occured event
+async function getNecessarySessionEvents({
   projectId,
   sessionId,
-  hoursInterval,
+  createdAt,
 }: {
   projectId: string;
   sessionId: string;
-  hoursInterval: number;
-}) {
+  createdAt: Date;
+}): Promise<ReturnType<typeof getEvents>> {
   const sql = `
     SELECT * FROM ${TABLE_NAMES.events} 
     WHERE 
       session_id = '${sessionId}' 
       AND project_id = '${projectId}'
-      AND created_at > now() - interval ${hoursInterval} HOUR
-    ORDER BY created_at DESC
+      AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'
+      AND (
+        name IN ('screen_view', 'session_start') 
+        OR created_at = (
+          SELECT MAX(created_at) 
+          FROM ${TABLE_NAMES.events} 
+          WHERE session_id = '${sessionId}' 
+          AND project_id = '${projectId}'
+          AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'
+          AND name NOT IN ('screen_view', 'session_start')
+        )
+      )
+    ORDER BY created_at DESC;
   `;
 
-  return getEvents(sql);
-}
-
-async function getCompleteSessionWithSessionStart({
-  projectId,
-  sessionId,
-  logger,
-}: {
-  projectId: string;
-  sessionId: string;
-  logger: ILogger;
-}): Promise<ReturnType<typeof getEvents>> {
-  const intervals = [1, 6, 12, 24, 72];
-  let intervalIndex = 0;
-  for (const hoursInterval of intervals) {
-    const events = await getCompleteSession({
+  const [lastScreenView, eventsInDb] = await Promise.all([
+    eventBuffer.getLastScreenView({
       projectId,
       sessionId,
-      hoursInterval,
-    });
+    }),
+    getEvents(sql),
+  ]);
 
-    if (events.find((event) => event.name === 'session_start')) {
-      return events;
-    }
-
-    const nextHoursInterval = intervals[++intervalIndex];
-    if (nextHoursInterval) {
-      logger.warn(`Checking last ${nextHoursInterval} hours for session_start`);
-    }
-  }
-
-  return [];
+  // sort last inserted first
+  return [lastScreenView, ...eventsInDb]
+    .filter((event): event is IServiceEvent => !!event)
+    .sort(
+      (a, b) =>
+        new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
+    );
 }
 
 export async function createSessionEnd(
@@ -77,56 +71,27 @@ export async function createSessionEnd(
 
   const payload = job.data.payload;
 
-  const [lastScreenView, eventsInDb] = await Promise.all([
-    eventBuffer.getLastScreenView({
-      projectId: payload.projectId,
-      sessionId: payload.sessionId,
-    }),
-    getCompleteSessionWithSessionStart({
-      projectId: payload.projectId,
-      sessionId: payload.sessionId,
-      logger,
-    }),
-  ]);
+  const events = await getNecessarySessionEvents({
+    projectId: payload.projectId,
+    sessionId: payload.sessionId,
+    createdAt: payload.createdAt,
+  });
 
-  // sort last inserted first
-  const events = [lastScreenView, ...eventsInDb]
-    .filter((event): event is IServiceEvent => !!event)
-    .sort(
-      (a, b) =>
-        new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
-    );
-
-  const sessionDuration = events.reduce((acc, event) => {
-    return acc + event.duration;
-  }, 0);
-
-  let sessionStart = events.find((event) => event.name === 'session_start');
-  const lastEvent = events[0];
+  const sessionStart = events.find((event) => event.name === 'session_start');
   const screenViews = events.filter((event) => event.name === 'screen_view');
+  const lastEvent = events[0];
 
   if (!sessionStart) {
-    const firstScreenView = last(screenViews);
-
-    if (!firstScreenView) {
-      throw new Error('Could not found session_start or any screen_view');
-    }
-
-    logger.warn('Creating session_start since it was not found');
-
-    sessionStart = {
-      ...firstScreenView,
-      name: 'session_start',
-      createdAt: new Date(getTime(firstScreenView.createdAt) - 100),
-    };
-
-    await createEvent(sessionStart);
+    throw new Error('No session_start found');
   }
 
   if (!lastEvent) {
     throw new Error('No last event found');
   }
 
+  const sessionDuration =
+    lastEvent.createdAt.getTime() - sessionStart.createdAt.getTime();
+
   await checkNotificationRulesForSessionEnd(events);
 
   logger.info('Creating session_end', {
@@ -135,7 +100,6 @@ export async function createSessionEnd(
     screenViews,
     sessionDuration,
     events,
-    lastScreenView: lastScreenView ? lastScreenView : 'none',
   });
 
   return createEvent({

apps/worker/src/jobs/events.incoming-event.ts

@@ -1,9 +1,10 @@
-import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
-import type { Job } from 'bullmq';
-import { omit } from 'ramda';
-
 import { logger as baseLogger } from '@/utils/logger';
-import { createSessionEnd, getSessionEnd } from '@/utils/session-handler';
+import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
+import {
+  createSessionEndJob,
+  createSessionStart,
+  getSessionEnd,
+} from '@/utils/session-handler';
 import { isSameDomain, parsePath } from '@openpanel/common';
 import { parseUserAgent } from '@openpanel/common/server';
 import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
@@ -14,7 +15,11 @@ import {
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
+import { getLock } from '@openpanel/redis';
+import { DelayedError, type Job } from 'bullmq';
+import { omit } from 'ramda';
 import * as R from 'ramda';
+import { v4 as uuid } from 'uuid';
 
 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
 
@@ -29,16 +34,18 @@ async function createEventAndNotify(
   jobData: Job<EventsQueuePayloadIncomingEvent>['data']['payload'],
   logger: ILogger,
 ) {
-  await checkNotificationRulesForEvent(payload).catch((e) => {
-    logger.error('Error checking notification rules', { error: e });
-  });
-
   logger.info('Creating event', { event: payload, jobData });
-
-  return createEvent(payload);
+  const [event] = await Promise.all([
+    createEvent(payload),
+    checkNotificationRulesForEvent(payload),
+  ]);
+  return event;
 }
 
-export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
+export async function incomingEvent(
+  job: Job<EventsQueuePayloadIncomingEvent>,
+  token?: string,
+) {
   const {
     geo,
     event: body,
@@ -46,7 +53,6 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
     projectId,
     currentDeviceId,
     previousDeviceId,
-    priority,
   } = job.data.payload;
   const properties = body.properties ?? {};
   const reqId = headers['request-id'] ?? 'unknown';
@@ -131,32 +137,50 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
   }
 
   const sessionEnd = await getSessionEnd({
-    priority,
     projectId,
     currentDeviceId,
     previousDeviceId,
     profileId,
   });
 
-  const lastScreenView = await eventBuffer.getLastScreenView({
-    projectId,
-    sessionId: sessionEnd.payload.sessionId,
-  });
+  const lastScreenView = sessionEnd
+    ? await eventBuffer.getLastScreenView({
+        projectId,
+        sessionId: sessionEnd.sessionId,
+      })
+    : null;
 
   const payload: IServiceCreateEventPayload = merge(baseEvent, {
-    deviceId: sessionEnd.payload.deviceId,
-    sessionId: sessionEnd.payload.sessionId,
-    referrer: sessionEnd.payload?.referrer,
-    referrerName: sessionEnd.payload?.referrerName,
-    referrerType: sessionEnd.payload?.referrerType,
+    deviceId: sessionEnd?.deviceId ?? currentDeviceId,
+    sessionId: sessionEnd?.sessionId ?? uuid(),
+    referrer: sessionEnd?.referrer ?? baseEvent.referrer,
+    referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,
+    referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,
     // if the path is not set, use the last screen view path
     path: baseEvent.path || lastScreenView?.path || '',
     origin: baseEvent.origin || lastScreenView?.origin || '',
   } as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;
 
-  if (sessionEnd.notFound) {
-    await createSessionEnd({ payload });
+  if (!sessionEnd) {
+    // Too avoid several created sessions we just throw if a lock exists
+    // This will than retry the job
+    const lock = await getLock(
+      `create-session-end:${currentDeviceId}`,
+      'locked',
+      1000,
+    );
+
+    if (!lock) {
+      logger.warn('Move incoming event to delayed');
+      await job.moveToDelayed(Date.now() + 50, token);
+      throw new DelayedError();
+    }
+    await createSessionStart({ payload });
   }
 
-  return createEventAndNotify(payload, job.data.payload, logger);
+  const event = await createEventAndNotify(payload, job.data.payload, logger);
+
+  await createSessionEndJob({ payload });
+
+  return event;
 }

apps/worker/src/jobs/events.ts

@@ -7,6 +7,9 @@ import type {
 
 import { incomingEvent } from './events.incoming-event';
 
-export async function eventsJob(job: Job<EventsQueuePayload>) {
-  return await incomingEvent(job as Job<EventsQueuePayloadIncomingEvent>);
+export async function eventsJob(job: Job<EventsQueuePayload>, token?: string) {
+  return await incomingEvent(
+    job as Job<EventsQueuePayloadIncomingEvent>,
+    token,
+  );
 }