fix: optimize event buffer (#278)

* fix: how we fetch profiles in the buffer

* perf: optimize event buffer

* remove unused file

* fix

* wip

* wip: try groupmq 2

* try simplified event buffer with duration calculation on the fly instead

apps/worker/src/boot-workers.ts

@@ -1,10 +1,9 @@
-import type { Queue, WorkerOptions } from 'bullmq';
-import { Worker } from 'bullmq';
-
+import { performance } from 'node:perf_hooks';
+import { setTimeout as sleep } from 'node:timers/promises';
 import {
+  cronQueue,
   EVENTS_GROUP_QUEUES_SHARDS,
   type EventsQueuePayloadIncomingEvent,
-  cronQueue,
   eventsGroupQueues,
   gscQueue,
   importQueue,
@@ -15,14 +14,12 @@ import {
   sessionsQueue,
 } from '@openpanel/queue';
 import { getRedisQueue } from '@openpanel/redis';
-
-import { performance } from 'node:perf_hooks';
-import { setTimeout as sleep } from 'node:timers/promises';
+import type { Queue, WorkerOptions } from 'bullmq';
+import { Worker } from 'bullmq';
 import { Worker as GroupWorker } from 'groupmq';
-
 import { cronJob } from './jobs/cron';
-import { gscJob } from './jobs/gsc';
 import { incomingEvent } from './jobs/events.incoming-event';
+import { gscJob } from './jobs/gsc';
 import { importJob } from './jobs/import';
 import { insightsProjectJob } from './jobs/insights';
 import { miscJob } from './jobs/misc';
@@ -95,7 +92,7 @@ function getConcurrencyFor(queueName: string, defaultValue = 1): number {
   return defaultValue;
 }
 
-export async function bootWorkers() {
+export function bootWorkers() {
   const enabledQueues = getEnabledQueues();
 
   const workers: (Worker | GroupWorker<any>)[] = [];
@@ -119,12 +116,14 @@ export async function bootWorkers() {
 
   for (const index of eventQueuesToStart) {
     const queue = eventsGroupQueues[index];
-    if (!queue) continue;
+    if (!queue) {
+      continue;
+    }
 
     const queueName = `events_${index}`;
     const concurrency = getConcurrencyFor(
       queueName,
-      Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10),
+      Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10)
     );
 
     const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
@@ -132,7 +131,7 @@ export async function bootWorkers() {
       concurrency,
       logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,
       blockingTimeoutSec: Number.parseFloat(
-        process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',
+        process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1'
       ),
       handler: async (job) => {
         return await incomingEvent(job.data);
@@ -172,7 +171,7 @@ export async function bootWorkers() {
     const notificationWorker = new Worker(
       notificationQueue.name,
       notificationJob,
-      { ...workerOptions, concurrency },
+      { ...workerOptions, concurrency }
     );
     workers.push(notificationWorker);
     logger.info('Started worker for notification', { concurrency });
@@ -224,7 +223,7 @@ export async function bootWorkers() {
 
   if (workers.length === 0) {
     logger.warn(
-      'No workers started. Check ENABLED_QUEUES environment variable.',
+      'No workers started. Check ENABLED_QUEUES environment variable.'
     );
   }
 
@@ -254,7 +253,7 @@ export async function bootWorkers() {
           const elapsed = job.finishedOn - job.processedOn;
           eventsGroupJobDuration.observe(
             { name: worker.name, status: 'failed' },
-            elapsed,
+            elapsed
           );
         }
         logger.error('job failed', {
@@ -267,23 +266,6 @@ export async function bootWorkers() {
       }
     });
 
-    (worker as Worker).on('completed', (job) => {
-      if (job) {
-        if (job.processedOn && job.finishedOn) {
-          const elapsed = job.finishedOn - job.processedOn;
-          logger.info('job completed', {
-            jobId: job.id,
-            worker: worker.name,
-            elapsed,
-          });
-          eventsGroupJobDuration.observe(
-            { name: worker.name, status: 'success' },
-            elapsed,
-          );
-        }
-      }
-    });
-
     (worker as Worker).on('ioredis:close', () => {
       logger.error('worker closed due to ioredis:close', {
         worker: worker.name,
@@ -293,7 +275,7 @@ export async function bootWorkers() {
 
   async function exitHandler(
     eventName: string,
-    evtOrExitCodeOrError: number | string | Error,
+    evtOrExitCodeOrError: number | string | Error
   ) {
     // Log the actual error details for unhandled rejections/exceptions
     if (evtOrExitCodeOrError instanceof Error) {
@@ -339,7 +321,7 @@ export async function bootWorkers() {
       process.on(evt, (code) => {
         exitHandler(evt, code);
       });
-    },
+    }
   );
 
   return workers;