feature(worker+api): improve buffer

2024-09-17 23:37:11 +02:00
parent 32e91959f6
commit 545cbfd520
18 changed files with 1484 additions and 432 deletions
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -6,6 +6,7 @@ import { Worker } from 'bullmq';
 import express from 'express';

 import { createInitialSalts } from '@openpanel/db';
+import type { CronQueueType } from '@openpanel/queue';
 import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';
 import { getRedisQueue } from '@openpanel/redis';

@@ -137,70 +138,75 @@ async function start() {
    }),
  );

-  await cronQueue.add(
-    'salt',
+  const jobs: {
+    name: string;
+    type: CronQueueType;
+    pattern: string | number;
+  }[] = [
    {
+      name: 'salt',
      type: 'salt',
-      payload: undefined,
+      pattern: '0 0 * * *',
    },
    {
-      jobId: 'salt',
-      repeat: {
-        utc: true,
-        pattern: '0 0 * * *',
-      },
-    },
-  );
-
-  await cronQueue.add(
-    'flush',
-    {
+      name: 'flush',
      type: 'flushEvents',
-      payload: undefined,
+      pattern: 1000 * 10,
    },
    {
-      jobId: 'flushEvents',
-      repeat: {
-        every: process.env.BATCH_INTERVAL
-          ? Number.parseInt(process.env.BATCH_INTERVAL, 10)
-          : 1000 * 10,
-      },
-    },
-  );
-
-  await cronQueue.add(
-    'flush',
-    {
+      name: 'flush',
      type: 'flushProfiles',
-      payload: undefined,
+      pattern: 1000 * 60 * 30,
    },
-    {
-      jobId: 'flushProfiles',
-      repeat: {
-        every: 2 * 1000,
-      },
-    },
-  );
+  ];

  if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
+    jobs.push({
+      name: 'ping',
+      type: 'ping',
+      pattern: '0 0 * * *',
+    });
+  }
+
+  // Add repeatable jobs
+  for (const job of jobs) {
    await cronQueue.add(
-      'ping',
+      job.name,
      {
-        type: 'ping',
+        type: job.type,
        payload: undefined,
      },
      {
-        jobId: 'ping',
-        repeat: {
-          pattern: '0 0 * * *',
-        },
+        jobId: job.type,
+        repeat:
+          typeof job.pattern === 'number'
+            ? {
+                every: job.pattern,
+              }
+            : {
+                pattern: job.pattern,
+              },
      },
    );
  }

+  // Remove outdated repeatable jobs
  const repeatableJobs = await cronQueue.getRepeatableJobs();
-
-  logger.info('Repeatable jobs', { repeatableJobs });
+  for (const repeatableJob of repeatableJobs) {
+    const match = jobs.find(
+      (job) => `${job.name}:${job.type}:::${job.pattern}` === repeatableJob.key,
+    );
+    if (match) {
+      logger.info('Repeatable job exists', {
+        key: repeatableJob.key,
+      });
+    } else {
+      logger.info('Removing repeatable job', {
+        key: repeatableJob.key,
+      });
+      cronQueue.removeRepeatableByKey(repeatableJob.key);
+    }
+  }

  await createInitialSalts();
 }
--- a/apps/worker/src/jobs/cron.ts
+++ b/apps/worker/src/jobs/cron.ts
@@ -12,10 +12,10 @@ export async function cronJob(job: Job<CronQueuePayload>) {
      return await salt();
    }
    case 'flushEvents': {
-      return await eventBuffer.flush();
+      return await eventBuffer.tryFlush();
    }
    case 'flushProfiles': {
-      return await profileBuffer.flush();
+      return await profileBuffer.tryFlush();
    }
    case 'ping': {
      return await ping();
--- a/apps/worker/src/jobs/events.create-session-end.ts
+++ b/apps/worker/src/jobs/events.create-session-end.ts
@@ -15,7 +15,7 @@ export async function createSessionEnd(
 ) {
  const payload = job.data.payload;
  const eventsInBuffer = await eventBuffer.findMany(
-    (item) => item.event.session_id === payload.sessionId,
+    (item) => item.session_id === payload.sessionId,
  );

  const sql = `
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -9,12 +9,14 @@ import type { IServiceCreateEventPayload } from '@openpanel/db';
 import { createEvent } from '@openpanel/db';
 import { getLastScreenViewFromProfileId } from '@openpanel/db/src/services/event.service';
 import { findJobByPrefix, sessionsQueue } from '@openpanel/queue';
-import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
+import type {
+  EventsQueuePayloadCreateSessionEnd,
+  EventsQueuePayloadIncomingEvent,
+} from '@openpanel/queue';
 import { getRedisQueue } from '@openpanel/redis';

 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
-const SESSION_TIMEOUT = 1000 * 60 * 30;
-const SESSION_END_TIMEOUT = SESSION_TIMEOUT + 1000;
+export const SESSION_TIMEOUT = 1000 * 60 * 30;

 export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
  const {
@@ -100,37 +102,14 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
    previousDeviceId,
  });

-  const sessionEndPayload = sessionEnd?.job.data.payload || {
-    sessionId: uuid(),
-    deviceId: currentDeviceId,
-    profileId,
-    projectId,
-  };
-
-  const sessionEndJobId =
-    sessionEnd?.job.id ??
-    `sessionEnd:${projectId}:${sessionEndPayload.deviceId}:${Date.now()}`;
-
-  if (sessionEnd) {
-    // If for some reason we have a session end job that is not a createSessionEnd job
-    if (sessionEnd.job.data.type !== 'createSessionEnd') {
-      throw new Error('Invalid session end job');
-    }
-
-    await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
-  } else {
-    await sessionsQueue.add(
-      'session',
-      {
-        type: 'createSessionEnd',
-        payload: sessionEndPayload,
-      },
-      {
-        delay: SESSION_END_TIMEOUT,
-        jobId: sessionEndJobId,
-      },
-    );
-  }
+  const sessionEndPayload =
+    sessionEnd?.job.data.payload ||
+    ({
+      sessionId: uuid(),
+      deviceId: currentDeviceId,
+      profileId,
+      projectId,
+    } satisfies EventsQueuePayloadCreateSessionEnd['payload']);

  const payload: IServiceCreateEventPayload = {
    name: body.name,
@@ -158,13 +137,46 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
    duration: 0,
    path: path,
    origin: origin,
-    referrer: referrer?.url,
-    referrerName: referrer?.name || utmReferrer?.name || '',
-    referrerType: referrer?.type || utmReferrer?.type || '',
+    referrer: sessionEndPayload.referrer || referrer?.url,
+    referrerName:
+      sessionEndPayload.referrerName ||
+      referrer?.name ||
+      utmReferrer?.name ||
+      '',
+    referrerType:
+      sessionEndPayload.referrerType ||
+      referrer?.type ||
+      utmReferrer?.type ||
+      '',
    sdkName,
    sdkVersion,
  };

+  const sessionEndJobId =
+    sessionEnd?.job.id ??
+    `sessionEnd:${projectId}:${sessionEndPayload.deviceId}:${getTime(createdAt)}`;
+
+  if (sessionEnd) {
+    // If for some reason we have a session end job that is not a createSessionEnd job
+    if (sessionEnd.job.data.type !== 'createSessionEnd') {
+      throw new Error('Invalid session end job');
+    }
+
+    await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
+  } else {
+    await sessionsQueue.add(
+      'session',
+      {
+        type: 'createSessionEnd',
+        payload,
+      },
+      {
+        delay: SESSION_TIMEOUT,
+        jobId: sessionEndJobId,
+      },
+    );
+  }
+
  if (!sessionEnd) {
    await createEvent({
      ...payload,
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -0,0 +1,351 @@
+import { type Mock, beforeEach, describe, expect, it, mock } from 'bun:test';
+import { getTime, toISOString } from '@openpanel/common';
+import type { Job } from 'bullmq';
+import { SESSION_TIMEOUT, incomingEvent } from './events.incoming-event';
+
+const projectId = 'test-project';
+const currentDeviceId = 'device-123';
+const previousDeviceId = 'device-456';
+const geo = {
+  country: 'US',
+  city: 'New York',
+  region: 'NY',
+  longitude: 0,
+  latitude: 0,
+};
+
+const createEvent = mock(() => {});
+const getLastScreenViewFromProfileId = mock();
+// // Mock dependencies
+mock.module('@openpanel/db', () => ({
+  createEvent,
+  getLastScreenViewFromProfileId,
+}));
+
+const sessionsQueue = { add: mock(() => Promise.resolve({})) };
+
+const findJobByPrefix = mock();
+
+mock.module('@openpanel/queue', () => ({
+  sessionsQueue,
+  findJobByPrefix,
+}));
+
+const getRedisQueue = mock(() => ({
+  keys: mock(() => Promise.resolve([])),
+}));
+
+mock.module('@openpanel/redis', () => ({
+  getRedisQueue,
+}));
+
+describe('incomingEvent', () => {
+  beforeEach(() => {
+    createEvent.mockClear();
+    findJobByPrefix.mockClear();
+    sessionsQueue.add.mockClear();
+    getLastScreenViewFromProfileId.mockClear();
+  });
+
+  it('should create a session start and an event', async () => {
+    const timestamp = new Date();
+    // Mock job data
+    const jobData = {
+      payload: {
+        geo,
+        event: {
+          name: 'test_event',
+          timestamp: timestamp.toISOString(),
+          properties: { __path: 'https://example.com/test' },
+        },
+        headers: {
+          'user-agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+          'openpanel-sdk-name': 'web',
+          'openpanel-sdk-version': '1.0.0',
+        },
+        projectId,
+        currentDeviceId,
+        previousDeviceId,
+        priority: true,
+      },
+    };
+
+    const job = { data: jobData } as Job;
+
+    // Execute the job
+    await incomingEvent(job);
+
+    const event = {
+      name: 'test_event',
+      deviceId: currentDeviceId,
+      // @ts-expect-error
+      sessionId: createEvent.mock.calls[1][0].sessionId,
+      profileId: '',
+      projectId,
+      properties: {
+        __hash: undefined,
+        __query: undefined,
+      },
+      createdAt: timestamp,
+      country: 'US',
+      city: 'New York',
+      region: 'NY',
+      longitude: 0,
+      latitude: 0,
+      os: 'Windows',
+      osVersion: '10',
+      browser: 'Chrome',
+      browserVersion: '91.0.4472.124',
+      device: 'desktop',
+      brand: '',
+      model: '',
+      duration: 0,
+      path: '/test',
+      origin: 'https://example.com',
+      referrer: '',
+      referrerName: '',
+      referrerType: 'unknown',
+      sdkName: 'web',
+      sdkVersion: '1.0.0',
+    };
+
+    expect(sessionsQueue.add.mock.calls[0]).toMatchObject([
+      'session',
+      {
+        type: 'createSessionEnd',
+        payload: event,
+      },
+      {
+        delay: SESSION_TIMEOUT,
+        jobId: `sessionEnd:${projectId}:${event.deviceId}:${timestamp.getTime()}`,
+      },
+    ]);
+
+    // Assertions
+    // Issue: https://github.com/oven-sh/bun/issues/10380
+    // expect(createEvent).toHaveBeenCalledWith(...)
+    expect(createEvent.mock.calls[0]).toMatchObject([
+      {
+        name: 'session_start',
+        deviceId: currentDeviceId,
+        sessionId: expect.stringMatching(
+          /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,
+        ),
+        profileId: '',
+        projectId,
+        properties: {
+          __hash: undefined,
+          __query: undefined,
+        },
+        createdAt: new Date(timestamp.getTime() - 100),
+        country: 'US',
+        city: 'New York',
+        region: 'NY',
+        longitude: 0,
+        latitude: 0,
+        os: 'Windows',
+        osVersion: '10',
+        browser: 'Chrome',
+        browserVersion: '91.0.4472.124',
+        device: 'desktop',
+        brand: '',
+        model: '',
+        duration: 0,
+        path: '/test',
+        origin: 'https://example.com',
+        referrer: '',
+        referrerName: '',
+        referrerType: 'unknown',
+        sdkName: 'web',
+        sdkVersion: '1.0.0',
+      },
+    ]);
+    expect(createEvent.mock.calls[1]).toMatchObject([event]);
+
+    // Add more specific assertions based on the expected behavior
+  });
+
+  it('should reuse existing session', async () => {
+    // Mock job data
+    const jobData = {
+      payload: {
+        geo,
+        event: {
+          name: 'test_event',
+          timestamp: new Date().toISOString(),
+          properties: { __path: 'https://example.com/test' },
+        },
+        headers: {
+          'user-agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+          'openpanel-sdk-name': 'web',
+          'openpanel-sdk-version': '1.0.0',
+        },
+        projectId,
+        currentDeviceId,
+        previousDeviceId,
+        priority: false,
+      },
+    };
+    const changeDelay = mock();
+    findJobByPrefix.mockReturnValueOnce({
+      changeDelay,
+      data: {
+        type: 'createSessionEnd',
+        payload: {
+          sessionId: 'session-123',
+          deviceId: currentDeviceId,
+          profileId: currentDeviceId,
+          projectId,
+        },
+      },
+    });
+
+    const job = { data: jobData } as Job;
+
+    // Execute the job
+    await incomingEvent(job);
+
+    expect(changeDelay.mock.calls[0]).toMatchObject([SESSION_TIMEOUT]);
+
+    // Assertions
+    // Issue: https://github.com/oven-sh/bun/issues/10380
+    // expect(createEvent).toHaveBeenCalledWith(...)
+    expect(createEvent.mock.calls[0]).toMatchObject([
+      {
+        name: 'test_event',
+        deviceId: currentDeviceId,
+        profileId: '',
+        sessionId: 'session-123',
+        projectId,
+        properties: {
+          __hash: undefined,
+          __query: undefined,
+        },
+        createdAt: expect.any(Date),
+        country: 'US',
+        city: 'New York',
+        region: 'NY',
+        longitude: 0,
+        latitude: 0,
+        os: 'Windows',
+        osVersion: '10',
+        browser: 'Chrome',
+        browserVersion: '91.0.4472.124',
+        device: 'desktop',
+        brand: '',
+        model: '',
+        duration: 0,
+        path: '/test',
+        origin: 'https://example.com',
+        referrer: '',
+        referrerName: '',
+        referrerType: 'unknown',
+        sdkName: 'web',
+        sdkVersion: '1.0.0',
+      },
+    ]);
+
+    // Add more specific assertions based on the expected behavior
+  });
+
+  it('should handle server events', async () => {
+    const timestamp = new Date();
+    const jobData = {
+      payload: {
+        geo,
+        event: {
+          name: 'server_event',
+          timestamp: timestamp.toISOString(),
+          properties: { custom_property: 'test_value' },
+          profileId: 'profile-123',
+        },
+        headers: {
+          'user-agent': 'OpenPanel Server/1.0',
+          'openpanel-sdk-name': 'server',
+          'openpanel-sdk-version': '1.0.0',
+        },
+        projectId,
+        currentDeviceId: '',
+        previousDeviceId: '',
+        priority: true,
+      },
+    };
+
+    const job = { data: jobData } as Job;
+
+    const mockLastScreenView = {
+      deviceId: 'last-device-123',
+      sessionId: 'last-session-456',
+      country: 'CA',
+      city: 'Toronto',
+      region: 'ON',
+      os: 'iOS',
+      osVersion: '15.0',
+      browser: 'Safari',
+      browserVersion: '15.0',
+      device: 'mobile',
+      brand: 'Apple',
+      model: 'iPhone',
+      path: '/last-path',
+      origin: 'https://example.com',
+      referrer: 'https://google.com',
+      referrerName: 'Google',
+      referrerType: 'search',
+    };
+
+    getLastScreenViewFromProfileId.mockReturnValueOnce(mockLastScreenView);
+
+    await incomingEvent(job);
+
+    // expect(getLastScreenViewFromProfileId).toHaveBeenCalledWith({
+    //   profileId: 'profile-123',
+    //   projectId,
+    // });
+
+    expect(createEvent.mock.calls[0]).toMatchObject([
+      {
+        name: 'server_event',
+        deviceId: 'last-device-123',
+        sessionId: 'last-session-456',
+        profileId: 'profile-123',
+        projectId,
+        properties: {
+          custom_property: 'test_value',
+          user_agent: 'OpenPanel Server/1.0',
+        },
+        createdAt: timestamp,
+        country: 'CA',
+        city: 'Toronto',
+        region: 'ON',
+        longitude: 0,
+        latitude: 0,
+        os: 'iOS',
+        osVersion: '15.0',
+        browser: 'Safari',
+        browserVersion: '15.0',
+        device: 'mobile',
+        brand: 'Apple',
+        model: 'iPhone',
+        duration: 0,
+        path: '/last-path',
+        origin: 'https://example.com',
+        referrer: 'https://google.com',
+        referrerName: 'Google',
+        referrerType: 'search',
+        sdkName: 'server',
+        sdkVersion: '1.0.0',
+      },
+    ]);
+
+    expect(sessionsQueue.add).not.toHaveBeenCalled();
+    expect(findJobByPrefix).not.toHaveBeenCalled();
+  });
+
+  // Add more test cases for different scenarios:
+  // - Server events
+  // - Existing sessions
+  // - Different priorities
+  // - Error cases
+});
--- a/apps/worker/src/utils/parse-user-agent.ts
+++ b/apps/worker/src/utils/parse-user-agent.ts
@@ -9,7 +9,9 @@ export function parseUserAgent(ua?: string | null) {
  if (!ua) return parsedServerUa;
  const res = new UAParser(ua).getResult();

-  if (isServer(ua)) return parsedServerUa;
+  if (isServer(ua)) {
+    return parsedServerUa;
+  }

  return {
    os: res.os.name,
@@ -77,7 +79,9 @@ function isServer(userAgent: string) {
    return true;
  }

-  return !!userAgent.match(/^([^\s]+\/[\d.]+\s*)+$/);
+  // Matches user agents like "Go-http-client/1.0" or "Go Http Client/1.0"
+  // It should just match the first name (with optional spaces) and version
+  return !!userAgent.match(/^[^\/]+\/[\d.]+$/);
 }

 export function getDevice(ua: string) {