improve(queue): how we handle incoming events and session ends

apps/api/src/controllers/event.controller.ts

@@ -55,15 +55,6 @@ export async function postEvent(
     return;
   }
 
-  const isScreenView = request.body.name === 'screen_view';
-  // this will ensure that we don't have multiple events creating sessions
-  const LOCK_DURATION = 1000;
-  const locked = await getLock(
-    `request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,
-    'locked',
-    LOCK_DURATION,
-  );
-
   await eventsQueue.add(
     'event',
     {
@@ -79,7 +70,6 @@ export async function postEvent(
         geo,
         currentDeviceId,
         previousDeviceId,
-        priority: locked,
       },
     },
     {
@@ -88,10 +78,6 @@ export async function postEvent(
         type: 'exponential',
         delay: 200,
       },
-      // Prioritize 'screen_view' events by setting no delay
-      // This ensures that session starts are created from 'screen_view' events
-      // rather than other events, maintaining accurate session tracking
-      delay: isScreenView ? undefined : LOCK_DURATION - 100,
     },
   );
 

apps/api/src/controllers/track.controller.ts

@@ -284,15 +284,6 @@ async function track({
   timestamp: string;
   isTimestampFromThePast: boolean;
 }) {
-  const isScreenView = payload.name === 'screen_view';
-  // this will ensure that we don't have multiple events creating sessions
-  const LOCK_DURATION = 1000;
-  const locked = await getLock(
-    `request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,
-    'locked',
-    LOCK_DURATION,
-  );
-
   await eventsQueue.add(
     'event',
     {
@@ -308,7 +299,6 @@ async function track({
         geo,
         currentDeviceId,
         previousDeviceId,
-        priority: locked,
       },
     },
     {
@@ -317,10 +307,6 @@ async function track({
         type: 'exponential',
         delay: 200,
       },
-      // Prioritize 'screen_view' events by setting no delay
-      // This ensures that session starts are created from 'screen_view' events
-      // rather than other events, maintaining accurate session tracking
-      delay: isScreenView ? undefined : LOCK_DURATION - 100,
     },
   );
 }

apps/api/tsup.config.ts

@@ -11,7 +11,7 @@ const options: Options = {
     '@node-rs/argon2',
     'bcrypt',
   ],
-  ignoreWatch: ['../../**/{.git,node_modules}/**'],
+  ignoreWatch: ['../../**/{.git,node_modules,dist}/**'],
   sourcemap: true,
   splitting: false,
 };

apps/worker/src/jobs/events.create-session-end.ts

@@ -1,5 +1,4 @@
 import type { Job } from 'bullmq';
-import { last } from 'ramda';
 
 import { logger as baseLogger } from '@/utils/logger';
 import { getTime } from '@openpanel/common';
@@ -9,61 +8,56 @@ import {
   checkNotificationRulesForSessionEnd,
   createEvent,
   eventBuffer,
+  formatClickhouseDate,
   getEvents,
 } from '@openpanel/db';
-import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';
 
-async function getCompleteSession({
+// Grabs session_start and screen_views + the last occured event
+async function getNecessarySessionEvents({
   projectId,
   sessionId,
-  hoursInterval,
+  createdAt,
 }: {
   projectId: string;
   sessionId: string;
-  hoursInterval: number;
+  createdAt: Date;
-}) {
+}): Promise<ReturnType<typeof getEvents>> {
   const sql = `
     SELECT * FROM ${TABLE_NAMES.events} 
     WHERE 
       session_id = '${sessionId}' 
       AND project_id = '${projectId}'
-      AND created_at > now() - interval ${hoursInterval} HOUR
+      AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'
-    ORDER BY created_at DESC
+      AND (
+        name IN ('screen_view', 'session_start') 
+        OR created_at = (
+          SELECT MAX(created_at) 
+          FROM ${TABLE_NAMES.events} 
+          WHERE session_id = '${sessionId}' 
+          AND project_id = '${projectId}'
+          AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'
+          AND name NOT IN ('screen_view', 'session_start')
+        )
+      )
+    ORDER BY created_at DESC;
   `;
 
-  return getEvents(sql);
+  const [lastScreenView, eventsInDb] = await Promise.all([
-}
+    eventBuffer.getLastScreenView({
-
-async function getCompleteSessionWithSessionStart({
-  projectId,
-  sessionId,
-  logger,
-}: {
-  projectId: string;
-  sessionId: string;
-  logger: ILogger;
-}): Promise<ReturnType<typeof getEvents>> {
-  const intervals = [1, 6, 12, 24, 72];
-  let intervalIndex = 0;
-  for (const hoursInterval of intervals) {
-    const events = await getCompleteSession({
       projectId,
       sessionId,
-      hoursInterval,
+    }),
-    });
+    getEvents(sql),
+  ]);
 
-    if (events.find((event) => event.name === 'session_start')) {
+  // sort last inserted first
-      return events;
+  return [lastScreenView, ...eventsInDb]
-    }
+    .filter((event): event is IServiceEvent => !!event)
-
+    .sort(
-    const nextHoursInterval = intervals[++intervalIndex];
+      (a, b) =>
-    if (nextHoursInterval) {
+        new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
-      logger.warn(`Checking last ${nextHoursInterval} hours for session_start`);
+    );
-    }
-  }
-
-  return [];
 }
 
 export async function createSessionEnd(
@@ -77,56 +71,27 @@ export async function createSessionEnd(
 
   const payload = job.data.payload;
 
-  const [lastScreenView, eventsInDb] = await Promise.all([
+  const events = await getNecessarySessionEvents({
-    eventBuffer.getLastScreenView({
+    projectId: payload.projectId,
-      projectId: payload.projectId,
+    sessionId: payload.sessionId,
-      sessionId: payload.sessionId,
+    createdAt: payload.createdAt,
-    }),
+  });
-    getCompleteSessionWithSessionStart({
-      projectId: payload.projectId,
-      sessionId: payload.sessionId,
-      logger,
-    }),
-  ]);
 
-  // sort last inserted first
+  const sessionStart = events.find((event) => event.name === 'session_start');
-  const events = [lastScreenView, ...eventsInDb]
-    .filter((event): event is IServiceEvent => !!event)
-    .sort(
-      (a, b) =>
-        new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
-    );
-
-  const sessionDuration = events.reduce((acc, event) => {
-    return acc + event.duration;
-  }, 0);
-
-  let sessionStart = events.find((event) => event.name === 'session_start');
-  const lastEvent = events[0];
   const screenViews = events.filter((event) => event.name === 'screen_view');
+  const lastEvent = events[0];
 
   if (!sessionStart) {
-    const firstScreenView = last(screenViews);
+    throw new Error('No session_start found');
-
-    if (!firstScreenView) {
-      throw new Error('Could not found session_start or any screen_view');
-    }
-
-    logger.warn('Creating session_start since it was not found');
-
-    sessionStart = {
-      ...firstScreenView,
-      name: 'session_start',
-      createdAt: new Date(getTime(firstScreenView.createdAt) - 100),
-    };
-
-    await createEvent(sessionStart);
   }
 
   if (!lastEvent) {
     throw new Error('No last event found');
   }
 
+  const sessionDuration =
+    lastEvent.createdAt.getTime() - sessionStart.createdAt.getTime();
+
   await checkNotificationRulesForSessionEnd(events);
 
   logger.info('Creating session_end', {
@@ -135,7 +100,6 @@ export async function createSessionEnd(
     screenViews,
     sessionDuration,
     events,
-    lastScreenView: lastScreenView ? lastScreenView : 'none',
   });
 
   return createEvent({

apps/worker/src/jobs/events.incoming-event.ts

@@ -1,9 +1,10 @@
-import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
-import type { Job } from 'bullmq';
-import { omit } from 'ramda';
-
 import { logger as baseLogger } from '@/utils/logger';
-import { createSessionEnd, getSessionEnd } from '@/utils/session-handler';
+import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
+import {
+  createSessionEndJob,
+  createSessionStart,
+  getSessionEnd,
+} from '@/utils/session-handler';
 import { isSameDomain, parsePath } from '@openpanel/common';
 import { parseUserAgent } from '@openpanel/common/server';
 import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
@@ -14,7 +15,11 @@ import {
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
+import { getLock } from '@openpanel/redis';
+import { DelayedError, type Job } from 'bullmq';
+import { omit } from 'ramda';
 import * as R from 'ramda';
+import { v4 as uuid } from 'uuid';
 
 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
 
@@ -29,16 +34,18 @@ async function createEventAndNotify(
   jobData: Job<EventsQueuePayloadIncomingEvent>['data']['payload'],
   logger: ILogger,
 ) {
-  await checkNotificationRulesForEvent(payload).catch((e) => {
-    logger.error('Error checking notification rules', { error: e });
-  });
-
   logger.info('Creating event', { event: payload, jobData });
-
+  const [event] = await Promise.all([
-  return createEvent(payload);
+    createEvent(payload),
+    checkNotificationRulesForEvent(payload),
+  ]);
+  return event;
 }
 
-export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
+export async function incomingEvent(
+  job: Job<EventsQueuePayloadIncomingEvent>,
+  token?: string,
+) {
   const {
     geo,
     event: body,
@@ -46,7 +53,6 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
     projectId,
     currentDeviceId,
     previousDeviceId,
-    priority,
   } = job.data.payload;
   const properties = body.properties ?? {};
   const reqId = headers['request-id'] ?? 'unknown';
@@ -131,32 +137,50 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
   }
 
   const sessionEnd = await getSessionEnd({
-    priority,
     projectId,
     currentDeviceId,
     previousDeviceId,
     profileId,
   });
 
-  const lastScreenView = await eventBuffer.getLastScreenView({
+  const lastScreenView = sessionEnd
-    projectId,
+    ? await eventBuffer.getLastScreenView({
-    sessionId: sessionEnd.payload.sessionId,
+        projectId,
-  });
+        sessionId: sessionEnd.sessionId,
+      })
+    : null;
 
   const payload: IServiceCreateEventPayload = merge(baseEvent, {
-    deviceId: sessionEnd.payload.deviceId,
+    deviceId: sessionEnd?.deviceId ?? currentDeviceId,
-    sessionId: sessionEnd.payload.sessionId,
+    sessionId: sessionEnd?.sessionId ?? uuid(),
-    referrer: sessionEnd.payload?.referrer,
+    referrer: sessionEnd?.referrer ?? baseEvent.referrer,
-    referrerName: sessionEnd.payload?.referrerName,
+    referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,
-    referrerType: sessionEnd.payload?.referrerType,
+    referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,
     // if the path is not set, use the last screen view path
     path: baseEvent.path || lastScreenView?.path || '',
     origin: baseEvent.origin || lastScreenView?.origin || '',
   } as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;
 
-  if (sessionEnd.notFound) {
+  if (!sessionEnd) {
-    await createSessionEnd({ payload });
+    // Too avoid several created sessions we just throw if a lock exists
+    // This will than retry the job
+    const lock = await getLock(
+      `create-session-end:${currentDeviceId}`,
+      'locked',
+      1000,
+    );
+
+    if (!lock) {
+      logger.warn('Move incoming event to delayed');
+      await job.moveToDelayed(Date.now() + 50, token);
+      throw new DelayedError();
+    }
+    await createSessionStart({ payload });
   }
 
-  return createEventAndNotify(payload, job.data.payload, logger);
+  const event = await createEventAndNotify(payload, job.data.payload, logger);
+
+  await createSessionEndJob({ payload });
+
+  return event;
 }

apps/worker/src/jobs/events.ts

@@ -7,6 +7,9 @@ import type {
 
 import { incomingEvent } from './events.incoming-event';
 
-export async function eventsJob(job: Job<EventsQueuePayload>) {
+export async function eventsJob(job: Job<EventsQueuePayload>, token?: string) {
-  return await incomingEvent(job as Job<EventsQueuePayloadIncomingEvent>);
+  return await incomingEvent(
+    job as Job<EventsQueuePayloadIncomingEvent>,
+    token,
+  );
 }

apps/worker/src/utils/session-handler.ts

@@ -3,22 +3,33 @@ import { type IServiceCreateEventPayload, createEvent } from '@openpanel/db';
 import {
   type EventsQueuePayloadCreateSessionEnd,
   sessionsQueue,
-  sessionsQueueEvents,
 } from '@openpanel/queue';
 import type { Job } from 'bullmq';
-import { v4 as uuid } from 'uuid';
+import { logger } from './logger';
 
 export const SESSION_TIMEOUT = 1000 * 60 * 30;
 
 const getSessionEndJobId = (projectId: string, deviceId: string) =>
   `sessionEnd:${projectId}:${deviceId}`;
 
-export async function createSessionEnd({
+export async function createSessionStart({
   payload,
 }: {
   payload: IServiceCreateEventPayload;
 }) {
-  await sessionsQueue.add(
+  return createEvent({
+    ...payload,
+    name: 'session_start',
+    createdAt: new Date(getTime(payload.createdAt) - 100),
+  });
+}
+
+export async function createSessionEndJob({
+  payload,
+}: {
+  payload: IServiceCreateEventPayload;
+}) {
+  return sessionsQueue.add(
     'session',
     {
       type: 'createSessionEnd',
@@ -34,12 +45,6 @@ export async function createSessionEnd({
       },
     },
   );
-
-  await createEvent({
-    ...payload,
-    name: 'session_start',
-    createdAt: new Date(getTime(payload.createdAt) - 100),
-  });
 }
 
 export async function getSessionEnd({
@@ -47,42 +52,33 @@ export async function getSessionEnd({
   currentDeviceId,
   previousDeviceId,
   profileId,
-  priority,
 }: {
   projectId: string;
   currentDeviceId: string;
   previousDeviceId: string;
   profileId: string;
-  priority: boolean;
 }) {
   const sessionEnd = await getSessionEndJob({
-    priority,
     projectId,
     currentDeviceId,
     previousDeviceId,
   });
 
-  const sessionEndPayload =
-    sessionEnd?.job.data.payload ||
-    ({
-      sessionId: uuid(),
-      deviceId: currentDeviceId,
-      profileId,
-      projectId,
-    } satisfies EventsQueuePayloadCreateSessionEnd['payload']);
-
   if (sessionEnd) {
-    // If for some reason we have a session end job that is not a createSessionEnd job
+    // Hack: if session end job just got created, we want to give it a chance to complete
-    if (sessionEnd.job.data.type !== 'createSessionEnd') {
+    // So the order is correct
-      throw new Error('Invalid session end job');
+    if (sessionEnd.job.timestamp > Date.now() - 50) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
     }
 
-    // If the profile_id is set and it's different from the device_id, we need to update the profile_id
+    const existingSessionIsAnonymous =
-    if (
-      sessionEnd.job.data.payload.profileId !== profileId &&
       sessionEnd.job.data.payload.profileId ===
-        sessionEnd.job.data.payload.deviceId
+      sessionEnd.job.data.payload.deviceId;
-    ) {
+
+    const eventIsIdentified =
+      sessionEnd.job.data.payload.profileId !== profileId;
+
+    if (existingSessionIsAnonymous && eventIsIdentified) {
       await sessionEnd.job.updateData({
         ...sessionEnd.job.data,
         payload: {
@@ -93,25 +89,22 @@ export async function getSessionEnd({
     }
 
     await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
+    return sessionEnd.job.data.payload;
   }
 
-  return {
+  return null;
-    payload: sessionEndPayload,
-    notFound: !sessionEnd,
-  };
 }
 
 export async function getSessionEndJob(args: {
   projectId: string;
   currentDeviceId: string;
   previousDeviceId: string;
-  priority: boolean;
   retryCount?: number;
 }): Promise<{
   deviceId: string;
   job: Job<EventsQueuePayloadCreateSessionEnd>;
 } | null> {
-  const { priority, retryCount = 0 } = args;
+  const { retryCount = 0 } = args;
 
   if (retryCount >= 6) {
     throw new Error('Failed to get session end');
@@ -125,46 +118,32 @@ export async function getSessionEndJob(args: {
     job: Job<EventsQueuePayloadCreateSessionEnd>;
   } | null> {
     const state = await job.getState();
-    if (state === 'delayed') {
+    if (state !== 'delayed') {
+      logger.info(`[session-handler] Session end job is in "${state}" state`, {
+        state,
+        retryCount,
+        jobTimestamp: new Date(job.timestamp).toISOString(),
+        jobDelta: Date.now() - job.timestamp,
+        jobId: job.id,
+        reqId: job.data.payload.properties?.__reqId ?? 'unknown',
+        payload: job.data.payload,
+      });
+    }
+
+    if (state === 'delayed' || state === 'waiting') {
       return { deviceId, job };
     }
 
-    if (state === 'failed') {
+    if (state === 'active') {
-      await job.retry();
+      await new Promise((resolve) => setTimeout(resolve, 100));
-      await job.waitUntilFinished(sessionsQueueEvents, 1000 * 10);
       return getSessionEndJob({
         ...args,
-        priority,
+        retryCount: retryCount + 1,
-        retryCount,
       });
     }
 
     if (state === 'completed') {
       await job.remove();
-      return getSessionEndJob({
-        ...args,
-        priority,
-        retryCount,
-      });
-    }
-
-    if (state === 'active' || state === 'waiting') {
-      await job.waitUntilFinished(sessionsQueueEvents, 1000 * 10);
-      return getSessionEndJob({
-        ...args,
-        priority,
-        retryCount,
-      });
-    }
-
-    // Shady state here, just remove it and retry
-    if (state === 'unknown') {
-      await job.remove();
-      return getSessionEndJob({
-        ...args,
-        priority,
-        retryCount,
-      });
     }
 
     return null;
@@ -175,8 +154,7 @@ export async function getSessionEndJob(args: {
     getSessionEndJobId(args.projectId, args.currentDeviceId),
   );
   if (currentJob) {
-    const res = await handleJobStates(currentJob, args.currentDeviceId);
+    return await handleJobStates(currentJob, args.currentDeviceId);
-    if (res) return res;
   }
 
   // Check previous device job
@@ -184,15 +162,7 @@ export async function getSessionEndJob(args: {
     getSessionEndJobId(args.projectId, args.previousDeviceId),
   );
   if (previousJob) {
-    const res = await handleJobStates(previousJob, args.previousDeviceId);
+    return await handleJobStates(previousJob, args.previousDeviceId);
-    if (res) return res;
-  }
-
-  // If no job found and not priority, retry
-  if (!priority) {
-    const backoffDelay = 50 * 2 ** retryCount;
-    await new Promise((resolve) => setTimeout(resolve, backoffDelay));
-    return getSessionEndJob({ ...args, priority, retryCount: retryCount + 1 });
   }
 
   // Create session

apps/worker/tsup.config.ts

@@ -6,7 +6,7 @@ const options: Options = {
   entry: ['src/index.ts'],
   noExternal: [/^@openpanel\/.*$/u, /^@\/.*$/u],
   external: ['@hyperdx/node-opentelemetry', 'winston'],
-  ignoreWatch: ['../../**/{.git,node_modules}/**'],
+  ignoreWatch: ['../../**/{.git,node_modules,dist}/**'],
   sourcemap: true,
   splitting: false,
 };

packages/db/src/buffers/base-buffer.ts

@@ -67,8 +67,6 @@ export class BaseBuffer {
           lockId,
         });
       }
-    } else {
-      this.logger.warn('Failed to acquire lock. Skipping flush.', { lockId });
     }
   }
 }

packages/db/src/buffers/session-buffer.ts

@@ -58,14 +58,17 @@ export class SessionBuffer extends BaseBuffer {
       if (event.origin) {
         newSession.exit_origin = event.origin;
       }
-      newSession.duration =
+      const duration =
         new Date(newSession.ended_at).getTime() -
         new Date(newSession.created_at).getTime();
-      if (newSession.duration < 0) {
+      if (duration > 0) {
+        newSession.duration = duration;
+      } else {
         this.logger.warn('Session duration is negative', {
+          duration,
+          event,
           session: newSession,
         });
-        newSession.duration = 0;
       }
       newSession.properties = toDots({
         ...(event.properties || {}),
@@ -73,7 +76,7 @@ export class SessionBuffer extends BaseBuffer {
       });
       // newSession.revenue += event.properties?.__revenue ?? 0;
 
-      if (event.name === 'screen_view') {
+      if (event.name === 'screen_view' && event.path) {
         newSession.screen_views.push(event.path);
         newSession.screen_view_count += 1;
       } else {
@@ -161,8 +164,6 @@ export class SessionBuffer extends BaseBuffer {
       const sessions = await this.getSession(event);
       const [newSession] = sessions;
 
-      console.log(`Adding sessions ${sessions.length}`);
-
       const multi = this.redis.multi();
       multi.set(
         `session:${newSession.id}`,

packages/db/src/services/event.service.ts

@@ -1,8 +1,8 @@
-import { path, assocPath, last, mergeDeepRight, pick, uniq } from 'ramda';
+import { path, assocPath, last, mergeDeepRight } from 'ramda';
 import { escape } from 'sqlstring';
 import { v4 as uuid } from 'uuid';
 
-import { toDots } from '@openpanel/common';
+import { DateTime, toDots } from '@openpanel/common';
 import { cacheable, getCache } from '@openpanel/redis';
 import type { IChartEventFilter } from '@openpanel/validation';
 
@@ -19,13 +19,8 @@ import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 import { createSqlBuilder } from '../sql-builder';
 import { getEventFiltersWhereClause } from './chart.service';
-import type { IClickhouseProfile, IServiceProfile } from './profile.service';
+import type { IServiceProfile } from './profile.service';
-import {
+import { getProfileById, getProfiles, upsertProfile } from './profile.service';
-  getProfileById,
-  getProfiles,
-  transformProfile,
-  upsertProfile,
-} from './profile.service';
 
 export type IImportedEvent = Omit<
   IClickhouseEvent,
@@ -293,6 +288,42 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
     payload.profileId = payload.deviceId;
   }
 
+  const event: IClickhouseEvent = {
+    id: uuid(),
+    name: payload.name,
+    device_id: payload.deviceId,
+    profile_id: payload.profileId ? String(payload.profileId) : '',
+    project_id: payload.projectId,
+    session_id: payload.sessionId,
+    properties: toDots(payload.properties),
+    path: payload.path ?? '',
+    origin: payload.origin ?? '',
+    created_at: DateTime.fromJSDate(payload.createdAt)
+      .setZone('UTC')
+      .toFormat('yyyy-MM-dd HH:mm:ss.SSS'),
+    country: payload.country ?? '',
+    city: payload.city ?? '',
+    region: payload.region ?? '',
+    longitude: payload.longitude ?? null,
+    latitude: payload.latitude ?? null,
+    os: payload.os ?? '',
+    os_version: payload.osVersion ?? '',
+    browser: payload.browser ?? '',
+    browser_version: payload.browserVersion ?? '',
+    device: payload.device ?? '',
+    brand: payload.brand ?? '',
+    model: payload.model ?? '',
+    duration: payload.duration,
+    referrer: payload.referrer ?? '',
+    referrer_name: payload.referrerName ?? '',
+    referrer_type: payload.referrerType ?? '',
+    imported_at: null,
+    sdk_name: payload.sdkName ?? '',
+    sdk_version: payload.sdkVersion ?? '',
+  };
+
+  await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);
+
   if (payload.profileId) {
     const profile = {
       id: String(payload.profileId),
@@ -326,40 +357,6 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
     }
   }
 
-  const event: IClickhouseEvent = {
-    id: uuid(),
-    name: payload.name,
-    device_id: payload.deviceId,
-    profile_id: payload.profileId ? String(payload.profileId) : '',
-    project_id: payload.projectId,
-    session_id: payload.sessionId,
-    properties: toDots(payload.properties),
-    path: payload.path ?? '',
-    origin: payload.origin ?? '',
-    created_at: formatClickhouseDate(payload.createdAt),
-    country: payload.country ?? '',
-    city: payload.city ?? '',
-    region: payload.region ?? '',
-    longitude: payload.longitude ?? null,
-    latitude: payload.latitude ?? null,
-    os: payload.os ?? '',
-    os_version: payload.osVersion ?? '',
-    browser: payload.browser ?? '',
-    browser_version: payload.browserVersion ?? '',
-    device: payload.device ?? '',
-    brand: payload.brand ?? '',
-    model: payload.model ?? '',
-    duration: payload.duration,
-    referrer: payload.referrer ?? '',
-    referrer_name: payload.referrerName ?? '',
-    referrer_type: payload.referrerType ?? '',
-    imported_at: null,
-    sdk_name: payload.sdkName ?? '',
-    sdk_version: payload.sdkVersion ?? '',
-  };
-
-  await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);
-
   return {
     document: event,
   };

packages/queue/src/queues.ts

@@ -22,14 +22,18 @@ export interface EventsQueuePayloadIncomingEvent {
     headers: Record<string, string | undefined>;
     currentDeviceId: string;
     previousDeviceId: string;
-    priority: boolean;
   };
 }
 export interface EventsQueuePayloadCreateEvent {
   type: 'createEvent';
   payload: Omit<IServiceEvent, 'id'>;
 }
-type SessionEndRequired = 'sessionId' | 'deviceId' | 'profileId' | 'projectId';
+type SessionEndRequired =
+  | 'sessionId'
+  | 'deviceId'
+  | 'profileId'
+  | 'projectId'
+  | 'createdAt';
 export interface EventsQueuePayloadCreateSessionEnd {
   type: 'createSessionEnd';
   payload: Partial<Omit<IServiceEvent, SessionEndRequired>> &

pnpm-lock.yaml

@@ -4,6 +4,12 @@ settings:
   autoInstallPeers: true
   excludeLinksFromLockfile: false
 
+catalogs:
+  default:
+    zod:
+      specifier: ^3.24.2
+      version: 3.24.2
+
 importers:
 
   .:
@@ -1066,6 +1072,40 @@ importers:
         specifier: ^5.2.2
         version: 5.6.3
 
+  packages/fire:
+    dependencies:
+      '@faker-js/faker':
+        specifier: ^9.0.1
+        version: 9.0.1
+      '@openpanel/common':
+        specifier: workspace:*
+        version: link:../common
+      '@openpanel/db':
+        specifier: workspace:*
+        version: link:../db
+      csv-parse:
+        specifier: ^5.6.0
+        version: 5.6.0
+      date-fns:
+        specifier: ^3.3.1
+        version: 3.3.1
+    devDependencies:
+      '@openpanel/tsconfig':
+        specifier: workspace:*
+        version: link:../../tooling/typescript
+      '@openpanel/validation':
+        specifier: workspace:*
+        version: link:../validation
+      '@types/node':
+        specifier: 20.14.8
+        version: 20.14.8
+      tsup:
+        specifier: ^7.2.0
+        version: 7.3.0(postcss@8.5.3)(typescript@5.6.3)
+      typescript:
+        specifier: ^5.2.2
+        version: 5.6.3
+
   packages/integrations:
     dependencies:
       '@slack/bolt':
@@ -3155,6 +3195,7 @@ packages:
   '@faker-js/faker@9.0.1':
     resolution: {integrity: sha512-4mDeYIgM3By7X6t5E6eYwLAa+2h4DeZDF7thhzIg6XB76jeEvMwadYAMCFJL/R4AnEBcAUO9+gL0vhy3s+qvZA==}
     engines: {node: '>=18.0.0', npm: '>=9.0.0'}
+    deprecated: Please update to a newer version
 
   '@fastify/accept-negotiator@2.0.1':
     resolution: {integrity: sha512-/c/TW2bO/v9JeEgoD/g1G5GxGeCF1Hafdf79WPmUlgYiBXummY0oX3VVq4yFkKKVBKDNlaDUYoab7g38RpPqCQ==}
@@ -7485,6 +7526,9 @@ packages:
   csstype@3.1.3:
     resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}
 
+  csv-parse@5.6.0:
+    resolution: {integrity: sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==}
+
   d3-array@2.12.1:
     resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==}
 
@@ -19905,6 +19949,8 @@ snapshots:
 
   csstype@3.1.3: {}
 
+  csv-parse@5.6.0: {}
+
   d3-array@2.12.1:
     dependencies:
       internmap: 1.0.1