feat: use groupmq instead of bullmq for incoming events (#206)

* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events

packages/queue/src/queues.ts

@@ -1,8 +1,12 @@
 import { Queue, QueueEvents } from 'bullmq';
 
-import type { IServiceEvent, Notification, Prisma } from '@openpanel/db';
-import { getRedisQueue } from '@openpanel/redis';
+import type { IServiceEvent, Prisma } from '@openpanel/db';
+import { createLogger } from '@openpanel/logger';
+import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
 import type { TrackPayload } from '@openpanel/sdk';
+import { Queue as GroupQueue } from 'groupmq';
+
+export const queueLogger = createLogger({ name: 'queue' });
 
 export interface EventsQueuePayloadIncomingEvent {
   type: 'incomingEvent';
@@ -103,6 +107,17 @@ export const eventsQueue = new Queue<EventsQueuePayload>('events', {
   },
 });
 
+export const eventsGroupQueue = new GroupQueue<
+  EventsQueuePayloadIncomingEvent['payload']
+>({
+  logger: queueLogger,
+  namespace: 'group_events',
+  redis: getRedisGroupQueue(),
+  orderingDelayMs: 2000,
+  keepCompleted: 10,
+  keepFailed: 10_000,
+});
+
 export const sessionsQueue = new Queue<SessionsQueuePayload>('sessions', {
   connection: getRedisQueue(),
   defaultJobOptions: {