feat: use groupmq instead of bullmq for incoming events (#206)

* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events

apps/worker/package.json

@@ -15,14 +15,15 @@
     "@bull-board/express": "5.21.0",
     "@openpanel/common": "workspace:*",
     "@openpanel/db": "workspace:*",
+    "@openpanel/email": "workspace:*",
     "@openpanel/integrations": "workspace:^",
     "@openpanel/json": "workspace:*",
     "@openpanel/logger": "workspace:*",
     "@openpanel/queue": "workspace:*",
     "@openpanel/redis": "workspace:*",
-    "@openpanel/email": "workspace:*",
     "bullmq": "^5.8.7",
     "express": "^4.18.2",
+    "groupmq": "1.0.0-next.13",
     "prom-client": "^15.1.3",
     "ramda": "^0.29.1",
     "source-map-support": "^0.5.21",

apps/worker/src/boot-workers.ts

@@ -2,18 +2,24 @@ import type { Queue, WorkerOptions } from 'bullmq';
 import { Worker } from 'bullmq';
 
 import {
+  type EventsQueuePayloadIncomingEvent,
   cronQueue,
+  eventsGroupQueue,
   eventsQueue,
   miscQueue,
   notificationQueue,
+  queueLogger,
   sessionsQueue,
 } from '@openpanel/queue';
 import { getRedisQueue } from '@openpanel/redis';
 
 import { performance } from 'node:perf_hooks';
 import { setTimeout as sleep } from 'node:timers/promises';
+import { Worker as GroupWorker } from 'groupmq';
+
 import { cronJob } from './jobs/cron';
 import { eventsJob } from './jobs/events';
+import { incomingEventPure } from './jobs/events.incoming-event';
 import { miscJob } from './jobs/misc';
 import { notificationJob } from './jobs/notification';
 import { sessionsJob } from './jobs/sessions';
@@ -21,10 +27,24 @@ import { logger } from './utils/logger';
 
 const workerOptions: WorkerOptions = {
   connection: getRedisQueue(),
-  concurrency: Number.parseInt(process.env.CONCURRENCY || '1', 10),
 };
 
 export async function bootWorkers() {
+  const eventsGroupWorker = new GroupWorker<
+    EventsQueuePayloadIncomingEvent['payload']
+  >({
+    concurrency: Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '1', 10),
+    logger: queueLogger,
+    queue: eventsGroupQueue,
+    handler: async (job) => {
+      logger.info('processing event (group queue)', {
+        groupId: job.groupId,
+        timestamp: job.data.event.timestamp,
+      });
+      await incomingEventPure(job.data);
+    },
+  });
+  eventsGroupWorker.run();
   const eventsWorker = new Worker(eventsQueue.name, eventsJob, workerOptions);
   const sessionsWorker = new Worker(
     sessionsQueue.name,
@@ -45,29 +65,30 @@ export async function bootWorkers() {
     cronWorker,
     notificationWorker,
     miscWorker,
+    eventsGroupWorker,
   ];
 
   workers.forEach((worker) => {
-    worker.on('error', (error) => {
+    (worker as Worker).on('error', (error) => {
       logger.error('worker error', {
         worker: worker.name,
         error,
       });
     });
 
-    worker.on('closed', () => {
+    (worker as Worker).on('closed', () => {
       logger.info('worker closed', {
         worker: worker.name,
       });
     });
 
-    worker.on('ready', () => {
+    (worker as Worker).on('ready', () => {
       logger.info('worker ready', {
         worker: worker.name,
       });
     });
 
-    worker.on('failed', (job) => {
+    (worker as Worker).on('failed', (job) => {
       if (job) {
         logger.error('job failed', {
           worker: worker.name,
@@ -78,7 +99,7 @@ export async function bootWorkers() {
       }
     });
 
-    worker.on('completed', (job) => {
+    (worker as Worker).on('completed', (job) => {
       if (job) {
         logger.info('job completed', {
           worker: worker.name,
@@ -91,7 +112,7 @@ export async function bootWorkers() {
       }
     });
 
-    worker.on('ioredis:close', () => {
+    (worker as Worker).on('ioredis:close', () => {
       logger.error('worker closed due to ioredis:close', {
         worker: worker.name,
       });

apps/worker/src/index.ts

@@ -6,6 +6,7 @@ import express from 'express';
 import { createInitialSalts } from '@openpanel/db';
 import {
   cronQueue,
+  eventsGroupQueue,
   eventsQueue,
   miscQueue,
   notificationQueue,
@@ -13,6 +14,7 @@ import {
 } from '@openpanel/queue';
 import client from 'prom-client';
 
+import { BullBoardGroupMQAdapter } from 'groupmq';
 import sourceMapSupport from 'source-map-support';
 import { bootCron } from './boot-cron';
 import { bootWorkers } from './boot-workers';
@@ -33,6 +35,7 @@ async function start() {
     serverAdapter.setBasePath('/');
     createBullBoard({
       queues: [
+        new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
         new BullMQAdapter(eventsQueue),
         new BullMQAdapter(sessionsQueue),
         new BullMQAdapter(cronQueue),

apps/worker/src/jobs/events.incoming-event.ts

@@ -45,6 +45,14 @@ async function createEventAndNotify(
 export async function incomingEvent(
   job: Job<EventsQueuePayloadIncomingEvent>,
   token?: string,
+) {
+  return incomingEventPure(job.data.payload, job, token);
+}
+
+export async function incomingEventPure(
+  jobPayload: EventsQueuePayloadIncomingEvent['payload'],
+  job?: Job<EventsQueuePayloadIncomingEvent>,
+  token?: string,
 ) {
   const {
     geo,
@@ -53,7 +61,7 @@ export async function incomingEvent(
     projectId,
     currentDeviceId,
     previousDeviceId,
-  } = job.data.payload;
+  } = jobPayload;
   const properties = body.properties ?? {};
   const reqId = headers['request-id'] ?? 'unknown';
   const logger = baseLogger.child({
@@ -151,11 +159,7 @@ export async function incomingEvent(
       origin: screenView?.origin ?? baseEvent.origin,
     };
 
-    return createEventAndNotify(
-      payload as IServiceEvent,
-      job.data.payload,
-      logger,
-    );
+    return createEventAndNotify(payload as IServiceEvent, jobPayload, logger);
   }
 
   const sessionEnd = await getSessionEnd({
@@ -186,21 +190,22 @@ export async function incomingEvent(
   if (!sessionEnd) {
     // Too avoid several created sessions we just throw if a lock exists
     // This will than retry the job
-    const lock = await getLock(
-      `create-session-end:${currentDeviceId}`,
-      'locked',
-      1000,
-    );
+    if (job) {
+      const lock = await getLock(
+        `create-session-end:${currentDeviceId}`,
+        'locked',
+        1000,
+      );
 
-    if (!lock) {
-      logger.warn('Move incoming event to delayed');
-      await job.moveToDelayed(Date.now() + 50, token);
-      throw new DelayedError();
+      if (!lock) {
+        await job.moveToDelayed(Date.now() + 50, token);
+        throw new DelayedError();
+      }
     }
     await createSessionStart({ payload });
   }
 
-  const event = await createEventAndNotify(payload, job.data.payload, logger);
+  const event = await createEventAndNotify(payload, jobPayload, logger);
 
   if (!sessionEnd) {
     await createSessionEndJob({ payload });

apps/worker/src/metrics.ts

@@ -7,13 +7,18 @@ import {
   profileBuffer,
   sessionBuffer,
 } from '@openpanel/db';
-import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';
+import {
+  cronQueue,
+  eventsGroupQueue,
+  eventsQueue,
+  sessionsQueue,
+} from '@openpanel/queue';
 
 const Registry = client.Registry;
 
 export const register = new Registry();
 
-const queues = [eventsQueue, sessionsQueue, cronQueue];
+const queues = [eventsQueue, sessionsQueue, cronQueue, eventsGroupQueue];
 
 queues.forEach((queue) => {
   register.registerMetric(