feat: session replay

* wip

* wip

* wip

* wip

* final fixes

* comments

* fix

packages/db/src/buffers/index.ts

@@ -2,6 +2,7 @@ import { BotBuffer as BotBufferRedis } from './bot-buffer';
 import { EventBuffer as EventBufferRedis } from './event-buffer';
 import { ProfileBackfillBuffer } from './profile-backfill-buffer';
 import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
+import { ReplayBuffer } from './replay-buffer';
 import { SessionBuffer } from './session-buffer';
 
 export const eventBuffer = new EventBufferRedis();
@@ -9,5 +10,7 @@ export const profileBuffer = new ProfileBufferRedis();
 export const botBuffer = new BotBufferRedis();
 export const sessionBuffer = new SessionBuffer();
 export const profileBackfillBuffer = new ProfileBackfillBuffer();
+export const replayBuffer = new ReplayBuffer();
 
 export type { ProfileBackfillEntry } from './profile-backfill-buffer';
+export type { IClickhouseSessionReplayChunk } from './replay-buffer';

packages/db/src/buffers/replay-buffer.ts

@@ -0,0 +1,92 @@
+import { getSafeJson } from '@openpanel/json';
+import { getRedisCache } from '@openpanel/redis';
+import { TABLE_NAMES, ch } from '../clickhouse/client';
+import { BaseBuffer } from './base-buffer';
+
+export interface IClickhouseSessionReplayChunk {
+  project_id: string;
+  session_id: string;
+  chunk_index: number;
+  started_at: string;
+  ended_at: string;
+  events_count: number;
+  is_full_snapshot: boolean;
+  payload: string;
+}
+
+export class ReplayBuffer extends BaseBuffer {
+  private batchSize = process.env.REPLAY_BUFFER_BATCH_SIZE
+    ? Number.parseInt(process.env.REPLAY_BUFFER_BATCH_SIZE, 10)
+    : 500;
+  private chunkSize = process.env.REPLAY_BUFFER_CHUNK_SIZE
+    ? Number.parseInt(process.env.REPLAY_BUFFER_CHUNK_SIZE, 10)
+    : 500;
+
+  private readonly redisKey = 'replay-buffer';
+
+  constructor() {
+    super({
+      name: 'replay',
+      onFlush: async () => {
+        await this.processBuffer();
+      },
+    });
+  }
+
+  async add(chunk: IClickhouseSessionReplayChunk) {
+    try {
+      const redis = getRedisCache();
+      const result = await redis
+        .multi()
+        .rpush(this.redisKey, JSON.stringify(chunk))
+        .incr(this.bufferCounterKey)
+        .llen(this.redisKey)
+        .exec();
+
+      const bufferLength = (result?.[2]?.[1] as number) ?? 0;
+      if (bufferLength >= this.batchSize) {
+        await this.tryFlush();
+      }
+    } catch (error) {
+      this.logger.error('Failed to add replay chunk to buffer', { error });
+    }
+  }
+
+  async processBuffer() {
+    const redis = getRedisCache();
+    try {
+      const items = await redis.lrange(this.redisKey, 0, this.batchSize - 1);
+
+      if (items.length === 0) {
+        return;
+      }
+
+      const chunks = items
+        .map((item) => getSafeJson<IClickhouseSessionReplayChunk>(item))
+        .filter((item): item is IClickhouseSessionReplayChunk => item != null);
+
+      for (const chunk of this.chunks(chunks, this.chunkSize)) {
+        await ch.insert({
+          table: TABLE_NAMES.session_replay_chunks,
+          values: chunk,
+          format: 'JSONEachRow',
+        });
+      }
+
+      await redis
+        .multi()
+        .ltrim(this.redisKey, items.length, -1)
+        .decrby(this.bufferCounterKey, items.length)
+        .exec();
+
+      this.logger.debug('Processed replay chunks', { count: items.length });
+    } catch (error) {
+      this.logger.error('Failed to process replay buffer', { error });
+    }
+  }
+
+  async getBufferSize() {
+    const redis = getRedisCache();
+    return this.getBufferSizeWithCounter(() => redis.llen(this.redisKey));
+  }
+}

packages/db/src/buffers/session-buffer.ts

@@ -1,8 +1,7 @@
-import { type Redis, getRedisCache } from '@openpanel/redis';
-
 import { getSafeJson } from '@openpanel/json';
+import { getRedisCache, type Redis } from '@openpanel/redis';
 import { assocPath, clone } from 'ramda';
-import { TABLE_NAMES, ch } from '../clickhouse/client';
+import { ch, TABLE_NAMES } from '../clickhouse/client';
 import type { IClickhouseEvent } from '../services/event.service';
 import type { IClickhouseSession } from '../services/session.service';
 import { BaseBuffer } from './base-buffer';
@@ -35,14 +34,14 @@ export class SessionBuffer extends BaseBuffer {
       | {
           projectId: string;
           profileId: string;
-        },
+        }
   ) {
     let hit: string | null = null;
     if ('sessionId' in options) {
       hit = await this.redis.get(`session:${options.sessionId}`);
     } else {
       hit = await this.redis.get(
-        `session:${options.projectId}:${options.profileId}`,
+        `session:${options.projectId}:${options.profileId}`
       );
     }
 
@@ -54,7 +53,7 @@ export class SessionBuffer extends BaseBuffer {
   }
 
   async getSession(
-    event: IClickhouseEvent,
+    event: IClickhouseEvent
   ): Promise<[IClickhouseSession] | [IClickhouseSession, IClickhouseSession]> {
     const existingSession = await this.getExistingSession({
       sessionId: event.session_id,
@@ -186,14 +185,14 @@ export class SessionBuffer extends BaseBuffer {
         `session:${newSession.id}`,
         JSON.stringify(newSession),
         'EX',
-        60 * 60,
+        60 * 60
       );
       if (newSession.profile_id) {
         multi.set(
           `session:${newSession.project_id}:${newSession.profile_id}`,
           JSON.stringify(newSession),
           'EX',
-          60 * 60,
+          60 * 60
         );
       }
       for (const session of sessions) {
@@ -220,10 +219,12 @@ export class SessionBuffer extends BaseBuffer {
       const events = await this.redis.lrange(
         this.redisKey,
         0,
-        this.batchSize - 1,
+        this.batchSize - 1
       );
 
-      if (events.length === 0) return;
+      if (events.length === 0) {
+        return;
+      }
 
       const sessions = events
         .map((e) => getSafeJson<IClickhouseSession>(e))
@@ -258,7 +259,7 @@ export class SessionBuffer extends BaseBuffer {
     }
   }
 
-  async getBufferSize() {
+  getBufferSize() {
     return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
   }
 }