feat: group analytics

* wip

* wip

* wip

* wip

* wip

* add buffer

* wip

* wip

* fixes

* fix

* wip

* group validation

* fix group issues

* docs: add groups

apps/worker/src/boot-cron.ts

@@ -68,6 +68,11 @@ export async function bootCron() {
       type: 'flushReplay',
       pattern: 1000 * 10,
     },
+    {
+      name: 'flush',
+      type: 'flushGroups',
+      pattern: 1000 * 10,
+    },
     {
       name: 'insightsDaily',
       type: 'insightsDaily',

apps/worker/src/jobs/cron.ts

@@ -1,6 +1,6 @@
 import type { Job } from 'bullmq';
 
-import { eventBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
+import { eventBuffer, groupBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
 import type { CronQueuePayload } from '@openpanel/queue';
 
 import { jobdeleteProjects } from './cron.delete-projects';
@@ -30,6 +30,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
     case 'flushReplay': {
       return await replayBuffer.tryFlush();
     }
+    case 'flushGroups': {
+      return await groupBuffer.tryFlush();
+    }
     case 'ping': {
       return await ping();
     }

apps/worker/src/jobs/events.incoming-event.ts

@@ -134,6 +134,7 @@ export async function incomingEvent(
       __hash: hash,
       __query: query,
     }),
+    groups: body.groups ?? [],
     createdAt,
     duration: 0,
     sdkName,

apps/worker/src/jobs/events.incoming-events.test.ts

@@ -129,6 +129,7 @@ describe('incomingEvent', () => {
       referrerType: '',
       sdkName: jobData.headers['openpanel-sdk-name'],
       sdkVersion: jobData.headers['openpanel-sdk-version'],
+      groups: [],
     };
 
     (createEvent as Mock).mockReturnValue(event);
@@ -242,6 +243,7 @@ describe('incomingEvent', () => {
       referrerType: '',
       sdkName: jobData.headers['openpanel-sdk-name'],
       sdkVersion: jobData.headers['openpanel-sdk-version'],
+      groups: [],
     };
 
     expect(spySessionsQueueAdd).toHaveBeenCalledTimes(0);
@@ -312,6 +314,7 @@ describe('incomingEvent', () => {
       screen_views: [],
       sign: 1,
       version: 1,
+      groups: [],
     } satisfies IClickhouseSession);
 
     await incomingEvent(jobData);
@@ -349,6 +352,7 @@ describe('incomingEvent', () => {
       sdkName: 'server',
       sdkVersion: '1.0.0',
       revenue: undefined,
+      groups: [],
     });
 
     expect(sessionsQueue.add).not.toHaveBeenCalled();
@@ -412,6 +416,7 @@ describe('incomingEvent', () => {
       referrerType: undefined,
       sdkName: 'server',
       sdkVersion: '1.0.0',
+      groups: [],
     });
 
     expect(sessionsQueue.add).not.toHaveBeenCalled();