fix: overall perf improvements

* fix: ignore private ips * fix: performance related fixes * fix: simply event buffer * fix: default to 1 events queue shard * add: cleanup scripts * fix: comments * fix comments * fix * fix: groupmq * wip * fix: sync cachable * remove cluster names and add it behind env flag (if someone want to scale) * fix * wip * better logger * remove reqid and user agent * fix lock * remove wait_for_async_insert
2025-11-15 22:13:59 +01:00
parent 38cc53890a
commit da59622dce
66 changed files with 5042 additions and 3860 deletions
--- a/apps/worker/src/jobs/events.create-session-end.ts
+++ b/apps/worker/src/jobs/events.create-session-end.ts
@@ -1,13 +1,13 @@
 import type { Job } from 'bullmq';

 import { logger as baseLogger } from '@/utils/logger';
-import { getTime } from '@openpanel/common';
 import {
  type IClickhouseSession,
  type IServiceCreateEventPayload,
  type IServiceEvent,
  TABLE_NAMES,
  checkNotificationRulesForSessionEnd,
+  convertClickhouseDateToJs,
  createEvent,
  eventBuffer,
  formatClickhouseDate,
@@ -65,10 +65,9 @@ export async function createSessionEnd(
  const logger = baseLogger.child({
    payload,
    jobId: job.id,
-    reqId: payload.properties?.__reqId ?? 'unknown',
  });

-  logger.info('Processing session end job');
+  logger.debug('Processing session end job');

  const session = await sessionBuffer.getExistingSession(payload.sessionId);

@@ -77,7 +76,7 @@ export async function createSessionEnd(
  }

  try {
-    handleSessionEndNotifications({
+    await handleSessionEndNotifications({
      session,
      payload,
    });
@@ -103,7 +102,9 @@ export async function createSessionEnd(
    name: 'session_end',
    duration: session.duration ?? 0,
    path: lastScreenView?.path ?? '',
-    createdAt: new Date(getTime(session.ended_at) + 1000),
+    createdAt: new Date(
+      convertClickhouseDateToJs(session.ended_at).getTime() + 100,
+    ),
    profileId: lastScreenView?.profileId || payload.profileId,
  });
 }
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -18,9 +18,7 @@ import {
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
-import type { Job } from 'bullmq';
 import * as R from 'ramda';
-import { omit } from 'ramda';
 import { v4 as uuid } from 'uuid';

 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
@@ -33,10 +31,9 @@ const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>

 async function createEventAndNotify(
  payload: IServiceCreateEventPayload,
-  jobData: Job<EventsQueuePayloadIncomingEvent>['data']['payload'],
  logger: ILogger,
 ) {
-  logger.info('Creating event', { event: payload, jobData });
+  logger.info('Creating event', { event: payload });
  const [event] = await Promise.all([
    createEvent(payload),
    checkNotificationRulesForEvent(payload).catch(() => {}),
@@ -45,16 +42,7 @@ async function createEventAndNotify(
 }

 export async function incomingEvent(
-  job: Job<EventsQueuePayloadIncomingEvent>,
-  token?: string,
-) {
-  return incomingEventPure(job.data.payload, job, token);
-}
-
-export async function incomingEventPure(
  jobPayload: EventsQueuePayloadIncomingEvent['payload'],
-  job?: Job<EventsQueuePayloadIncomingEvent>,
-  token?: string,
 ) {
  const {
    geo,
@@ -63,6 +51,7 @@ export async function incomingEventPure(
    projectId,
    currentDeviceId,
    previousDeviceId,
+    uaInfo: _uaInfo,
  } = jobPayload;
  const properties = body.properties ?? {};
  const reqId = headers['request-id'] ?? 'unknown';
@@ -93,18 +82,17 @@ export async function incomingEventPure(
  const userAgent = headers['user-agent'];
  const sdkName = headers['openpanel-sdk-name'];
  const sdkVersion = headers['openpanel-sdk-version'];
-  const uaInfo = parseUserAgent(userAgent, properties);
+  // TODO: Remove both user-agent and parseUserAgent
+  const uaInfo = _uaInfo ?? parseUserAgent(userAgent, properties);

  const baseEvent = {
    name: body.name,
    profileId,
    projectId,
-    properties: omit(GLOBAL_PROPERTIES, {
+    properties: R.omit(GLOBAL_PROPERTIES, {
      ...properties,
-      __user_agent: userAgent,
      __hash: hash,
      __query: query,
-      __reqId: reqId,
    }),
    createdAt,
    duration: 0,
@@ -161,7 +149,7 @@ export async function incomingEventPure(
      origin: screenView?.origin ?? baseEvent.origin,
    };

-    return createEventAndNotify(payload as IServiceEvent, jobPayload, logger);
+    return createEventAndNotify(payload as IServiceEvent, logger);
  }

  const sessionEnd = await getSessionEnd({
@@ -197,7 +185,7 @@ export async function incomingEventPure(
    });
  }

-  const event = await createEventAndNotify(payload, jobPayload, logger);
+  const event = await createEventAndNotify(payload, logger);

  if (!sessionEnd) {
    logger.info('Creating session end job', { event: payload });
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -1,6 +1,9 @@
 import { type IServiceEvent, createEvent } from '@openpanel/db';
 import { eventBuffer } from '@openpanel/db';
-import { sessionsQueue } from '@openpanel/queue';
+import {
+  type EventsQueuePayloadIncomingEvent,
+  sessionsQueue,
+} from '@openpanel/queue';
 import type { Job } from 'bullmq';
 import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest';
 import { incomingEvent } from './events.incoming-event';
@@ -32,6 +35,28 @@ const geo = {
  latitude: 0,
 };

+const uaInfo: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
+  isServer: false,
+  device: 'desktop',
+  os: 'Windows',
+  osVersion: '10',
+  browser: 'Chrome',
+  browserVersion: '91.0.4472.124',
+  brand: '',
+  model: '',
+};
+
+const uaInfoServer: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
+  isServer: true,
+  device: 'server',
+  os: '',
+  osVersion: '',
+  browser: '',
+  browserVersion: '',
+  brand: '',
+  model: '',
+};
+
 describe('incomingEvent', () => {
  beforeEach(() => {
    vi.clearAllMocks();
@@ -41,31 +66,29 @@ describe('incomingEvent', () => {
    const spySessionsQueueAdd = vi.spyOn(sessionsQueue, 'add');
    const timestamp = new Date();
    // Mock job data
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'test_event',
-          timestamp: timestamp.toISOString(),
-          properties: { __path: 'https://example.com/test' },
-        },
-        headers: {
-          'request-id': '123',
-          'user-agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-          'openpanel-sdk-name': 'web',
-          'openpanel-sdk-version': '1.0.0',
-        },
-        projectId,
-        currentDeviceId,
-        previousDeviceId,
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'test_event',
+        timestamp: timestamp.toISOString(),
+        isTimestampFromThePast: false,
+        properties: { __path: 'https://example.com/test' },
      },
+      uaInfo,
+      headers: {
+        'request-id': '123',
+        'user-agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'openpanel-sdk-name': 'web',
+        'openpanel-sdk-version': '1.0.0',
+      },
+      projectId,
+      currentDeviceId,
+      previousDeviceId,
    };

-    const job = { data: jobData } as Job;
-
    // Execute the job
-    await incomingEvent(job);
+    await incomingEvent(jobData);

    const event = {
      name: 'test_event',
@@ -78,8 +101,6 @@ describe('incomingEvent', () => {
      properties: {
        __hash: undefined,
        __query: undefined,
-        __user_agent: jobData.payload.headers['user-agent'],
-        __reqId: jobData.payload.headers['request-id'],
      },
      createdAt: timestamp,
      country: 'US',
@@ -92,16 +113,16 @@ describe('incomingEvent', () => {
      browser: 'Chrome',
      browserVersion: '91.0.4472.124',
      device: 'desktop',
-      brand: undefined,
-      model: undefined,
+      brand: '',
+      model: '',
      duration: 0,
      path: '/test',
      origin: 'https://example.com',
      referrer: '',
      referrerName: '',
      referrerType: '',
-      sdkName: jobData.payload.headers['openpanel-sdk-name'],
-      sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
+      sdkName: jobData.headers['openpanel-sdk-name'],
+      sdkVersion: jobData.headers['openpanel-sdk-version'],
    };

    expect(spySessionsQueueAdd).toHaveBeenCalledWith(
@@ -135,29 +156,27 @@ describe('incomingEvent', () => {

    const timestamp = new Date();
    // Mock job data
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'test_event',
-          timestamp: timestamp.toISOString(),
-          properties: { __path: 'https://example.com/test' },
-        },
-        headers: {
-          'request-id': '123',
-          'user-agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-          'openpanel-sdk-name': 'web',
-          'openpanel-sdk-version': '1.0.0',
-        },
-        projectId,
-        currentDeviceId,
-        previousDeviceId,
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'test_event',
+        timestamp: timestamp.toISOString(),
+        properties: { __path: 'https://example.com/test' },
+        isTimestampFromThePast: false,
      },
+      headers: {
+        'request-id': '123',
+        'user-agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'openpanel-sdk-name': 'web',
+        'openpanel-sdk-version': '1.0.0',
+      },
+      uaInfo,
+      projectId,
+      currentDeviceId,
+      previousDeviceId,
    };

-    const job = { data: jobData } as Job;
-
    const changeDelay = vi.fn();
    const updateData = vi.fn();
    spySessionsQueueGetJob.mockResolvedValueOnce({
@@ -175,7 +194,7 @@ describe('incomingEvent', () => {
      },
    } as Partial<Job> as Job);
    // Execute the job
-    await incomingEvent(job);
+    await incomingEvent(jobData);

    const event = {
      name: 'test_event',
@@ -186,8 +205,6 @@ describe('incomingEvent', () => {
      properties: {
        __hash: undefined,
        __query: undefined,
-        __user_agent: jobData.payload.headers['user-agent'],
-        __reqId: jobData.payload.headers['request-id'],
      },
      createdAt: timestamp,
      country: 'US',
@@ -200,16 +217,16 @@ describe('incomingEvent', () => {
      browser: 'Chrome',
      browserVersion: '91.0.4472.124',
      device: 'desktop',
-      brand: undefined,
-      model: undefined,
+      brand: '',
+      model: '',
      duration: 0,
      path: '/test',
      origin: 'https://example.com',
      referrer: '',
      referrerName: '',
      referrerType: '',
-      sdkName: jobData.payload.headers['openpanel-sdk-name'],
-      sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
+      sdkName: jobData.headers['openpanel-sdk-name'],
+      sdkVersion: jobData.headers['openpanel-sdk-version'],
    };

    expect(spySessionsQueueAdd).toHaveBeenCalledTimes(0);
@@ -220,29 +237,27 @@ describe('incomingEvent', () => {

  it('should handle server events (with existing screen view)', async () => {
    const timestamp = new Date();
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'server_event',
-          timestamp: timestamp.toISOString(),
-          properties: { custom_property: 'test_value' },
-          profileId: 'profile-123',
-        },
-        headers: {
-          'user-agent': 'OpenPanel Server/1.0',
-          'openpanel-sdk-name': 'server',
-          'openpanel-sdk-version': '1.0.0',
-          'request-id': '123',
-        },
-        projectId,
-        currentDeviceId: '',
-        previousDeviceId: '',
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'server_event',
+        timestamp: timestamp.toISOString(),
+        properties: { custom_property: 'test_value' },
+        profileId: 'profile-123',
+        isTimestampFromThePast: false,
      },
+      headers: {
+        'user-agent': 'OpenPanel Server/1.0',
+        'openpanel-sdk-name': 'server',
+        'openpanel-sdk-version': '1.0.0',
+        'request-id': '123',
+      },
+      projectId,
+      currentDeviceId: '',
+      previousDeviceId: '',
+      uaInfo: uaInfoServer,
    };

-    const job = { data: jobData } as Job;
-
    const mockLastScreenView = {
      deviceId: 'last-device-123',
      sessionId: 'last-session-456',
@@ -268,7 +283,7 @@ describe('incomingEvent', () => {
      mockLastScreenView as IServiceEvent,
    );

-    await incomingEvent(job);
+    await incomingEvent(jobData);

    expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
      name: 'server_event',
@@ -278,8 +293,6 @@ describe('incomingEvent', () => {
      projectId,
      properties: {
        custom_property: 'test_value',
-        __user_agent: 'OpenPanel Server/1.0',
-        __reqId: '123',
        __hash: undefined,
        __query: undefined,
      },
@@ -311,33 +324,31 @@ describe('incomingEvent', () => {

  it('should handle server events (without existing screen view)', async () => {
    const timestamp = new Date();
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'server_event',
-          timestamp: timestamp.toISOString(),
-          properties: { custom_property: 'test_value' },
-          profileId: 'profile-123',
-        },
-        headers: {
-          'user-agent': 'OpenPanel Server/1.0',
-          'openpanel-sdk-name': 'server',
-          'openpanel-sdk-version': '1.0.0',
-          'request-id': '123',
-        },
-        projectId,
-        currentDeviceId: '',
-        previousDeviceId: '',
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'server_event',
+        timestamp: timestamp.toISOString(),
+        properties: { custom_property: 'test_value' },
+        profileId: 'profile-123',
+        isTimestampFromThePast: false,
      },
+      headers: {
+        'user-agent': 'OpenPanel Server/1.0',
+        'openpanel-sdk-name': 'server',
+        'openpanel-sdk-version': '1.0.0',
+        'request-id': '123',
+      },
+      projectId,
+      currentDeviceId: '',
+      previousDeviceId: '',
+      uaInfo: uaInfoServer,
    };

-    const job = { data: jobData } as Job;
-
    // Mock getLastScreenView to return null
    vi.mocked(eventBuffer.getLastScreenView).mockResolvedValueOnce(null);

-    await incomingEvent(job);
+    await incomingEvent(jobData);

    expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
      name: 'server_event',
@@ -347,8 +358,6 @@ describe('incomingEvent', () => {
      projectId,
      properties: {
        custom_property: 'test_value',
-        __user_agent: 'OpenPanel Server/1.0',
-        __reqId: '123',
        __hash: undefined,
        __query: undefined,
      },
--- a/apps/worker/src/jobs/events.ts
+++ b/apps/worker/src/jobs/events.ts
@@ -1,15 +0,0 @@
-import type { Job } from 'bullmq';
-
-import type {
-  EventsQueuePayload,
-  EventsQueuePayloadIncomingEvent,
-} from '@openpanel/queue';
-
-import { incomingEvent } from './events.incoming-event';
-
-export async function eventsJob(job: Job<EventsQueuePayload>, token?: string) {
-  return await incomingEvent(
-    job as Job<EventsQueuePayloadIncomingEvent>,
-    token,
-  );
-}