init redis lazy

packages/db/src/buffers/event-buffer.ts

@@ -2,7 +2,7 @@ import { groupBy } from 'ramda';
 import SuperJSON from 'superjson';
 
 import { deepMergeObjects } from '@openpanel/common';
-import { redis, redisPub } from '@openpanel/redis';
+import { getRedisCache, getRedisPub } from '@openpanel/redis';
 
 import { ch, TABLE_NAMES } from '../clickhouse-client';
 import { transformEvent } from '../services/event.service';
@@ -31,12 +31,12 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {
   constructor() {
     super({
       table: TABLE_NAMES.events,
-      redis,
+      redis: getRedisCache(),
     });
   }
 
   public onInsert?: OnInsert<IClickhouseEvent> | undefined = (event) => {
-    redisPub.publish(
+    getRedisPub().publish(
       'event:received',
       SuperJSON.stringify(transformEvent(event))
     );
@@ -54,7 +54,7 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {
     savedEvents
   ) => {
     for (const event of savedEvents) {
-      redisPub.publish(
+      getRedisPub().publish(
         'event:saved',
         SuperJSON.stringify(transformEvent(event))
       );

packages/db/src/buffers/profile-buffer.ts

@@ -1,7 +1,7 @@
 import { mergeDeepRight } from 'ramda';
 
 import { toDots } from '@openpanel/common';
-import { redis } from '@openpanel/redis';
+import { getRedisCache } from '@openpanel/redis';
 
 import { ch, chQuery } from '../clickhouse-client';
 import type {
@@ -22,7 +22,7 @@ import { RedisBuffer } from './buffer';
 export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {
   constructor() {
     super({
-      redis,
+      redis: getRedisCache(),
       table: 'profiles',
       batchSize: 100,
     });

packages/db/src/services/event.service.ts

@@ -3,7 +3,7 @@ import { escape } from 'sqlstring';
 import { v4 as uuid } from 'uuid';
 
 import { toDots } from '@openpanel/common';
-import { redis } from '@openpanel/redis';
+import { getRedisCache } from '@openpanel/redis';
 import type { IChartEventFilter } from '@openpanel/validation';
 
 import { eventBuffer } from '../buffers';
@@ -184,7 +184,7 @@ export function transformMinimalEvent(
 }
 
 export async function getLiveVisitors(projectId: string) {
-  const keys = await redis.keys(`live:event:${projectId}:*`);
+  const keys = await getRedisCache().keys(`live:event:${projectId}:*`);
   return keys.length;
 }
 

packages/queue/index.ts

@@ -1,5 +1,4 @@
 export { eventsQueue, cronQueue, sessionsQueue } from './src/queues';
 export type * from './src/queues';
-export { connection } from './src/connection';
 export { findJobByPrefix } from './src/utils';
 export type { JobsOptions } from 'bullmq';

packages/queue/package.json

@@ -9,6 +9,7 @@
   },
   "dependencies": {
     "@openpanel/db": "workspace:*",
+    "@openpanel/redis": "workspace:*",
     "bullmq": "^5.8.7"
   },
   "devDependencies": {

packages/queue/src/connection.ts

@@ -1,10 +0,0 @@
-const parse = (connectionString: string) => {
-  const url = new URL(connectionString);
-  return {
-    host: url.hostname,
-    port: Number(url.port),
-    password: url.password,
-  } as const;
-};
-
-export const connection = parse(String(process.env.REDIS_URL));

packages/queue/src/queues.ts

@@ -1,10 +1,9 @@
 import { Queue } from 'bullmq';
 
 import type { IServiceCreateEventPayload } from '@openpanel/db';
+import { getRedisQueue } from '@openpanel/redis';
 import type { PostEventPayload } from '@openpanel/sdk';
 
-import { connection } from './connection';
-
 export interface EventsQueuePayloadIncomingEvent {
   type: 'incomingEvent';
   payload: {
@@ -63,21 +62,21 @@ export type CronQueuePayload =
   | CronQueuePayloadFlushProfiles;
 
 export const eventsQueue = new Queue<EventsQueuePayload>('events', {
-  connection,
+  connection: getRedisQueue(),
   defaultJobOptions: {
     removeOnComplete: 10,
   },
 });
 
 export const sessionsQueue = new Queue<SessionsQueuePayload>('sessions', {
-  connection,
+  connection: getRedisQueue(),
   defaultJobOptions: {
     removeOnComplete: 10,
   },
 });
 
 export const cronQueue = new Queue<CronQueuePayload>('cron', {
-  connection,
+  connection: getRedisQueue(),
   defaultJobOptions: {
     removeOnComplete: 10,
   },

packages/redis/cachable.ts

@@ -1,4 +1,4 @@
-import { redis } from './redis';
+import { getRedisCache } from './redis';
 
 export function cacheable<T extends (...args: any) => any>(
   fn: T,
@@ -9,7 +9,7 @@ export function cacheable<T extends (...args: any) => any>(
   ): Promise<Awaited<ReturnType<T>>> {
     // JSON.stringify here is not bullet proof since ordering of object keys matters etc
     const key = `cachable:${fn.name}:${JSON.stringify(args)}`;
-    const cached = await redis.get(key);
+    const cached = await getRedisCache().get(key);
     if (cached) {
       try {
         return JSON.parse(cached);
@@ -20,7 +20,7 @@ export function cacheable<T extends (...args: any) => any>(
     const result = await fn(...(args as any));
 
     if (result !== undefined || result !== null) {
-      redis.setex(key, expire, JSON.stringify(result));
+      getRedisCache().setex(key, expire, JSON.stringify(result));
     }
 
     return result;

packages/redis/redis.ts

@@ -2,12 +2,65 @@ import type { RedisOptions } from 'ioredis';
 import Redis from 'ioredis';
 
 const options: RedisOptions = {
-  connectTimeout: 30000,
-  maxRetriesPerRequest: null,
+  connectTimeout: 10000,
 };
 
 export { Redis };
 
-export const redis = new Redis(process.env.REDIS_URL!, options);
-export const redisSub = new Redis(process.env.REDIS_URL!, options);
-export const redisPub = new Redis(process.env.REDIS_URL!, options);
+const createRedisClient = (
+  url: string,
+  overrides: RedisOptions = {}
+): Redis => {
+  const client = new Redis(url, { ...options, ...overrides });
+
+  client.on('error', (error) => {
+    console.error('Redis Client Error:', error);
+  });
+
+  return client;
+};
+
+let redisCache: Redis;
+export function getRedisCache() {
+  if (!redisCache) {
+    redisCache = createRedisClient(process.env.REDIS_URL!, options);
+  }
+
+  return redisCache;
+}
+
+let redisSub: Redis;
+export function getRedisSub() {
+  if (!redisSub) {
+    redisSub = createRedisClient(process.env.REDIS_URL!, options);
+  }
+
+  return redisSub;
+}
+
+let redisPub: Redis;
+export function getRedisPub() {
+  if (!redisPub) {
+    redisPub = createRedisClient(process.env.REDIS_URL!, options);
+  }
+
+  return redisPub;
+}
+
+let redisQueue: Redis;
+export function getRedisQueue() {
+  if (!redisQueue) {
+    // Use different redis for queues (self-hosting will re-use the same redis instance)
+    redisQueue = createRedisClient(
+      (process.env.QUEUE_REDIS_URL || process.env.REDIS_URL)!,
+      {
+        ...options,
+        enableReadyCheck: false,
+        maxRetriesPerRequest: null,
+        enableOfflineQueue: true,
+      }
+    );
+  }
+
+  return redisQueue;
+}