feat: use groupmq instead of bullmq for incoming events (#206)

* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events

apps/api/src/controllers/track.controller.ts

@@ -6,13 +6,14 @@ import { checkDuplicatedEvent } from '@/utils/deduplicate';
 import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
 import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
-import { eventsQueue } from '@openpanel/queue';
-import { getLock } from '@openpanel/redis';
+import { eventsGroupQueue, eventsQueue } from '@openpanel/queue';
+import { getLock, getRedisCache } from '@openpanel/redis';
 import type {
   DecrementPayload,
   IdentifyPayload,
   IncrementPayload,
   TrackHandlerPayload,
+  TrackPayload,
 } from '@openpanel/sdk';
 
 export function getStringHeaders(headers: FastifyRequest['headers']) {
@@ -260,11 +261,6 @@ export async function handler(
   reply.status(200).send();
 }
 
-type TrackPayload = {
-  name: string;
-  properties?: Record<string, any>;
-};
-
 async function track({
   payload,
   currentDeviceId,
@@ -284,11 +280,14 @@ async function track({
   timestamp: string;
   isTimestampFromThePast: boolean;
 }) {
-  await eventsQueue.add(
-    'event',
-    {
-      type: 'incomingEvent',
-      payload: {
+  const isGroupQueue = await getRedisCache().exists('group_queue');
+  if (isGroupQueue) {
+    const groupId = payload.profileId
+      ? `${projectId}:${payload.profileId}`
+      : currentDeviceId;
+    await eventsGroupQueue.add({
+      orderMs: new Date(timestamp).getTime(),
+      data: {
         projectId,
         headers,
         event: {
@@ -300,15 +299,35 @@ async function track({
         currentDeviceId,
         previousDeviceId,
       },
-    },
-    {
-      attempts: 3,
-      backoff: {
-        type: 'exponential',
-        delay: 200,
+      groupId,
+    });
+  } else {
+    await eventsQueue.add(
+      'event',
+      {
+        type: 'incomingEvent',
+        payload: {
+          projectId,
+          headers,
+          event: {
+            ...payload,
+            timestamp,
+            isTimestampFromThePast,
+          },
+          geo,
+          currentDeviceId,
+          previousDeviceId,
+        },
       },
-    },
-  );
+      {
+        attempts: 3,
+        backoff: {
+          type: 'exponential',
+          delay: 200,
+        },
+      },
+    );
+  }
 }
 
 async function identify({