feat: use groupmq instead of bullmq for incoming events (#206)

* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events

packages/redis/redis.ts

@@ -8,6 +8,8 @@ const options: RedisOptions = {
 
 export { Redis };
 
+const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
+
 export interface ExtendedRedis extends Redis {
   getJson: <T = any>(key: string) => Promise<T | null>;
   setJson: <T = any>(
@@ -63,7 +65,7 @@ const createRedisClient = (
 let redisCache: ExtendedRedis;
 export function getRedisCache() {
   if (!redisCache) {
-    redisCache = createRedisClient(process.env.REDIS_URL!, options);
+    redisCache = createRedisClient(REDIS_URL, options);
   }
 
   return redisCache;
@@ -72,7 +74,7 @@ export function getRedisCache() {
 let redisSub: ExtendedRedis;
 export function getRedisSub() {
   if (!redisSub) {
-    redisSub = createRedisClient(process.env.REDIS_URL!, options);
+    redisSub = createRedisClient(REDIS_URL, options);
   }
 
   return redisSub;
@@ -81,7 +83,7 @@ export function getRedisSub() {
 let redisPub: ExtendedRedis;
 export function getRedisPub() {
   if (!redisPub) {
-    redisPub = createRedisClient(process.env.REDIS_URL!, options);
+    redisPub = createRedisClient(REDIS_URL, options);
   }
 
   return redisPub;
@@ -91,20 +93,32 @@ let redisQueue: ExtendedRedis;
 export function getRedisQueue() {
   if (!redisQueue) {
     // Use different redis for queues (self-hosting will re-use the same redis instance)
-    redisQueue = createRedisClient(
-      (process.env.QUEUE_REDIS_URL || process.env.REDIS_URL)!,
-      {
-        ...options,
-        enableReadyCheck: false,
-        maxRetriesPerRequest: null,
-        enableOfflineQueue: true,
-      },
-    );
+    redisQueue = createRedisClient(REDIS_URL, {
+      ...options,
+      enableReadyCheck: false,
+      maxRetriesPerRequest: null,
+      enableOfflineQueue: true,
+    });
   }
 
   return redisQueue;
 }
 
+let redisGroupQueue: ExtendedRedis;
+export function getRedisGroupQueue() {
+  if (!redisGroupQueue) {
+    // Dedicated Redis connection for GroupWorker to avoid blocking BullMQ
+    redisGroupQueue = createRedisClient(REDIS_URL, {
+      ...options,
+      enableReadyCheck: false,
+      maxRetriesPerRequest: null,
+      enableOfflineQueue: true,
+    });
+  }
+
+  return redisGroupQueue;
+}
+
 export async function getLock(key: string, value: string, timeout: number) {
   const lock = await getRedisCache().set(key, value, 'PX', timeout, 'NX');
   return lock === 'OK';

packages/redis/run-every.ts

@@ -15,6 +15,6 @@ export async function runEvery({
     return;
   }
 
-  getRedisCache().set(cacheKey, 'true', 'EX', interval);
+  await getRedisCache().set(cacheKey, '1', 'EX', interval);
   return fn();
 }