wip

packages/db/code-migrations/10-add-session-replay.ts

@@ -0,0 +1,65 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { TABLE_NAMES } from '../src/clickhouse/client';
+import {
+  addColumns,
+  createTable,
+  modifyTTL,
+  runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+export async function up() {
+  const isClustered = getIsCluster();
+
+  const sqls: string[] = [
+    ...createTable({
+      name: TABLE_NAMES.session_replay_chunks,
+      columns: [
+        '`project_id` String CODEC(ZSTD(3))',
+        '`session_id` String CODEC(ZSTD(3))',
+        '`chunk_index` UInt16',
+        '`started_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+        '`ended_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+        '`events_count` UInt16',
+        '`is_full_snapshot` Bool',
+        '`payload` String CODEC(ZSTD(6))',
+      ],
+      orderBy: ['project_id', 'session_id', 'chunk_index'],
+      partitionBy: 'toYYYYMM(started_at)',
+      settings: {
+        index_granularity: 8192,
+      },
+      distributionHash: 'cityHash64(project_id, session_id)',
+      replicatedVersion: '1',
+      isClustered,
+    }),
+    ...addColumns(
+      TABLE_NAMES.sessions,
+      ['`has_replay` Bool DEFAULT 0'],
+      isClustered,
+    ),
+    modifyTTL({
+      tableName: TABLE_NAMES.session_replay_chunks,
+      isClustered,
+      ttl: 'started_at + INTERVAL 30 DAY',
+    }),
+  ];
+
+  fs.writeFileSync(
+    path.join(__filename.replace('.ts', '.sql')),
+    sqls
+      .map((sql) =>
+        sql
+          .trim()
+          .replace(/;$/, '')
+          .replace(/\n{2,}/g, '\n')
+          .concat(';'),
+      )
+      .join('\n\n---\n\n'),
+  );
+
+  if (!process.argv.includes('--dry')) {
+    await runClickhouseMigrationCommands(sqls);
+  }
+}

packages/db/code-migrations/migrate.ts

@@ -19,12 +19,19 @@ async function migrate() {
   const migration = args.filter((arg) => !arg.startsWith('--'))[0];
 
   const migrationsDir = path.join(__dirname, '..', 'code-migrations');
-  const migrations = fs.readdirSync(migrationsDir).filter((file) => {
-    const version = file.split('-')[0];
-    return (
-      !Number.isNaN(Number.parseInt(version ?? '')) && file.endsWith('.ts')
-    );
-  });
+  const migrations = fs
+    .readdirSync(migrationsDir)
+    .filter((file) => {
+      const version = file.split('-')[0];
+      return (
+        !Number.isNaN(Number.parseInt(version ?? '')) && file.endsWith('.ts')
+      );
+    })
+    .sort((a, b) => {
+      const aVersion = Number.parseInt(a.split('-')[0]!);
+      const bVersion = Number.parseInt(b.split('-')[0]!);
+      return aVersion - bVersion;
+    });
 
   const finishedMigrations = await db.codeMigration.findMany();
 

packages/db/src/buffers/session-buffer.ts

@@ -163,10 +163,46 @@ export class SessionBuffer extends BaseBuffer {
           : '',
         sign: 1,
         version: 1,
+        has_replay: false,
       },
     ];
   }
 
+  async markHasReplay(sessionId: string): Promise<void> {
+    console.log('markHasReplay', sessionId);
+    const existingSession = await this.getExistingSession({ sessionId });
+    if (!existingSession) {
+      console.log('no existing session or has replay', existingSession);
+      return;
+    }
+
+    if (existingSession.has_replay) {
+      return;
+    }
+
+    const oldSession = assocPath(['sign'], -1, clone(existingSession));
+    const newSession = assocPath(['sign'], 1, clone(existingSession));
+    newSession.version = existingSession.version + 1;
+    newSession.has_replay = true;
+
+    const multi = this.redis.multi();
+    multi.set(
+      `session:${sessionId}`,
+      JSON.stringify(newSession),
+      'EX',
+      60 * 60,
+    );
+    multi.rpush(this.redisKey, JSON.stringify(newSession));
+    multi.rpush(this.redisKey, JSON.stringify(oldSession));
+    multi.incrby(this.bufferCounterKey, 2);
+    await multi.exec();
+
+    const bufferLength = await this.getBufferSize();
+    if (bufferLength >= this.batchSize) {
+      await this.tryFlush();
+    }
+  }
+
   async add(event: IClickhouseEvent) {
     if (!event.session_id) {
       return;

packages/db/src/clickhouse/client.ts

@@ -59,6 +59,7 @@ export const TABLE_NAMES = {
   cohort_events_mv: 'cohort_events_mv',
   sessions: 'sessions',
   events_imports: 'events_imports',
+  session_replay_chunks: 'session_replay_chunks',
 };
 
 /**

packages/db/src/services/session.service.ts

@@ -52,6 +52,7 @@ export type IClickhouseSession = {
   revenue: number;
   sign: 1 | 0;
   version: number;
+  has_replay?: boolean;
 };
 
 export interface IServiceSession {
@@ -90,6 +91,7 @@ export interface IServiceSession {
   utmContent: string;
   utmTerm: string;
   revenue: number;
+  hasReplay: boolean;
   profile?: IServiceProfile;
 }
 
@@ -141,6 +143,7 @@ export function transformSession(session: IClickhouseSession): IServiceSession {
     utmContent: session.utm_content,
     utmTerm: session.utm_term,
     revenue: session.revenue,
+    hasReplay: session.has_replay ?? false,
     profile: undefined,
   };
 }
@@ -229,6 +232,7 @@ export async function getSessionList({
     'screen_view_count',
     'event_count',
     'revenue',
+    'has_replay',
   ];
 
   columns.forEach((column) => {
@@ -321,6 +325,41 @@ export async function getSessionsCount({
 
 export const getSessionsCountCached = cacheable(getSessionsCount, 60 * 10);
 
+export async function getSessionReplayEvents(
+  sessionId: string,
+  projectId: string,
+): Promise<{ events: unknown[] }> {
+  const chunks = await clix(ch)
+    .select<{ chunk_index: number; payload: string }>(['chunk_index', 'payload'])
+    .from(TABLE_NAMES.session_replay_chunks)
+    .where('session_id', '=', sessionId)
+    .where('project_id', '=', projectId)
+    .orderBy('chunk_index', 'ASC')
+    .execute();
+
+  const allEvents = chunks.flatMap((chunk) =>
+    JSON.parse(chunk.payload) as unknown[],
+  );
+
+  // rrweb event types: 2 = FullSnapshot, 4 = Meta
+  // Incremental snapshots (type 3) before the first FullSnapshot are orphaned
+  // and cause the player to fast-forward through empty time. Strip them but
+  // keep Meta events (type 4) since rrweb needs them for viewport dimensions.
+  const firstFullSnapshotIdx = allEvents.findIndex(
+    (e: any) => e.type === 2,
+  );
+
+  let events = allEvents;
+  if (firstFullSnapshotIdx > 0) {
+    const metaEvents = allEvents
+      .slice(0, firstFullSnapshotIdx)
+      .filter((e: any) => e.type === 4);
+    events = [...metaEvents, ...allEvents.slice(firstFullSnapshotIdx)];
+  }
+
+  return { events };
+}
+
 class SessionService {
   constructor(private client: typeof ch) {}