This commit is contained in:
Carl-Gerhard Lindesvärd
2026-02-09 09:54:22 +00:00
parent 38d9b65ec8
commit 551927af06
33 changed files with 1746 additions and 142 deletions

View File

@@ -4,6 +4,6 @@ export function shortId() {
return nanoid(4);
}
export function generateId() {
return nanoid(8);
export function generateId(prefix?: string, length?: number) {
return prefix ? `${prefix}_${nanoid(length ?? 8)}` : nanoid(length ?? 8);
}

View File

@@ -0,0 +1,65 @@
import fs from 'node:fs';
import path from 'node:path';
import { TABLE_NAMES } from '../src/clickhouse/client';
import {
addColumns,
createTable,
modifyTTL,
runClickhouseMigrationCommands,
} from '../src/clickhouse/migration';
import { getIsCluster } from './helpers';
export async function up() {
const isClustered = getIsCluster();
const sqls: string[] = [
...createTable({
name: TABLE_NAMES.session_replay_chunks,
columns: [
'`project_id` String CODEC(ZSTD(3))',
'`session_id` String CODEC(ZSTD(3))',
'`chunk_index` UInt16',
'`started_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
'`ended_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
'`events_count` UInt16',
'`is_full_snapshot` Bool',
'`payload` String CODEC(ZSTD(6))',
],
orderBy: ['project_id', 'session_id', 'chunk_index'],
partitionBy: 'toYYYYMM(started_at)',
settings: {
index_granularity: 8192,
},
distributionHash: 'cityHash64(project_id, session_id)',
replicatedVersion: '1',
isClustered,
}),
...addColumns(
TABLE_NAMES.sessions,
['`has_replay` Bool DEFAULT 0'],
isClustered,
),
modifyTTL({
tableName: TABLE_NAMES.session_replay_chunks,
isClustered,
ttl: 'started_at + INTERVAL 30 DAY',
}),
];
fs.writeFileSync(
path.join(__filename.replace('.ts', '.sql')),
sqls
.map((sql) =>
sql
.trim()
.replace(/;$/, '')
.replace(/\n{2,}/g, '\n')
.concat(';'),
)
.join('\n\n---\n\n'),
);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}

View File

@@ -19,12 +19,19 @@ async function migrate() {
const migration = args.filter((arg) => !arg.startsWith('--'))[0];
const migrationsDir = path.join(__dirname, '..', 'code-migrations');
const migrations = fs.readdirSync(migrationsDir).filter((file) => {
const version = file.split('-')[0];
return (
!Number.isNaN(Number.parseInt(version ?? '')) && file.endsWith('.ts')
);
});
const migrations = fs
.readdirSync(migrationsDir)
.filter((file) => {
const version = file.split('-')[0];
return (
!Number.isNaN(Number.parseInt(version ?? '')) && file.endsWith('.ts')
);
})
.sort((a, b) => {
const aVersion = Number.parseInt(a.split('-')[0]!);
const bVersion = Number.parseInt(b.split('-')[0]!);
return aVersion - bVersion;
});
const finishedMigrations = await db.codeMigration.findMany();

View File

@@ -163,10 +163,46 @@ export class SessionBuffer extends BaseBuffer {
: '',
sign: 1,
version: 1,
has_replay: false,
},
];
}
async markHasReplay(sessionId: string): Promise<void> {
console.log('markHasReplay', sessionId);
const existingSession = await this.getExistingSession({ sessionId });
if (!existingSession) {
console.log('no existing session or has replay', existingSession);
return;
}
if (existingSession.has_replay) {
return;
}
const oldSession = assocPath(['sign'], -1, clone(existingSession));
const newSession = assocPath(['sign'], 1, clone(existingSession));
newSession.version = existingSession.version + 1;
newSession.has_replay = true;
const multi = this.redis.multi();
multi.set(
`session:${sessionId}`,
JSON.stringify(newSession),
'EX',
60 * 60,
);
multi.rpush(this.redisKey, JSON.stringify(newSession));
multi.rpush(this.redisKey, JSON.stringify(oldSession));
multi.incrby(this.bufferCounterKey, 2);
await multi.exec();
const bufferLength = await this.getBufferSize();
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
}
async add(event: IClickhouseEvent) {
if (!event.session_id) {
return;

View File

@@ -59,6 +59,7 @@ export const TABLE_NAMES = {
cohort_events_mv: 'cohort_events_mv',
sessions: 'sessions',
events_imports: 'events_imports',
session_replay_chunks: 'session_replay_chunks',
};
/**

View File

@@ -52,6 +52,7 @@ export type IClickhouseSession = {
revenue: number;
sign: 1 | 0;
version: number;
has_replay?: boolean;
};
export interface IServiceSession {
@@ -90,6 +91,7 @@ export interface IServiceSession {
utmContent: string;
utmTerm: string;
revenue: number;
hasReplay: boolean;
profile?: IServiceProfile;
}
@@ -141,6 +143,7 @@ export function transformSession(session: IClickhouseSession): IServiceSession {
utmContent: session.utm_content,
utmTerm: session.utm_term,
revenue: session.revenue,
hasReplay: session.has_replay ?? false,
profile: undefined,
};
}
@@ -229,6 +232,7 @@ export async function getSessionList({
'screen_view_count',
'event_count',
'revenue',
'has_replay',
];
columns.forEach((column) => {
@@ -321,6 +325,41 @@ export async function getSessionsCount({
export const getSessionsCountCached = cacheable(getSessionsCount, 60 * 10);
export async function getSessionReplayEvents(
sessionId: string,
projectId: string,
): Promise<{ events: unknown[] }> {
const chunks = await clix(ch)
.select<{ chunk_index: number; payload: string }>(['chunk_index', 'payload'])
.from(TABLE_NAMES.session_replay_chunks)
.where('session_id', '=', sessionId)
.where('project_id', '=', projectId)
.orderBy('chunk_index', 'ASC')
.execute();
const allEvents = chunks.flatMap((chunk) =>
JSON.parse(chunk.payload) as unknown[],
);
// rrweb event types: 2 = FullSnapshot, 4 = Meta
// Incremental snapshots (type 3) before the first FullSnapshot are orphaned
// and cause the player to fast-forward through empty time. Strip them but
// keep Meta events (type 4) since rrweb needs them for viewport dimensions.
const firstFullSnapshotIdx = allEvents.findIndex(
(e: any) => e.type === 2,
);
let events = allEvents;
if (firstFullSnapshotIdx > 0) {
const metaEvents = allEvents
.slice(0, firstFullSnapshotIdx)
.filter((e: any) => e.type === 4);
events = [...metaEvents, ...allEvents.slice(firstFullSnapshotIdx)];
}
return { events };
}
class SessionService {
constructor(private client: typeof ch) {}

View File

@@ -65,8 +65,10 @@ export interface EventsQueuePayloadIncomingEvent {
latitude: number | undefined;
};
headers: Record<string, string | undefined>;
currentDeviceId: string;
previousDeviceId: string;
currentDeviceId: string; // TODO: Remove
previousDeviceId: string; // TODO: Remove
deviceId: string;
sessionId: string;
};
}
export interface EventsQueuePayloadCreateEvent {

View File

@@ -37,6 +37,8 @@ export type OpenPanelOptions = {
export class OpenPanel {
api: Api;
profileId?: string;
deviceId?: string;
sessionId?: string;
global?: Record<string, unknown>;
queue: TrackHandlerPayload[] = [];
@@ -69,6 +71,16 @@ export class OpenPanel {
this.flush();
}
private shouldQueue(payload: TrackHandlerPayload): boolean {
if (payload.type === 'replay' && !this.sessionId) {
return true;
}
if (this.options.waitForProfile && !this.profileId) {
return true;
}
return false;
}
async send(payload: TrackHandlerPayload) {
if (this.options.disabled) {
return Promise.resolve();
@@ -78,11 +90,25 @@ export class OpenPanel {
return Promise.resolve();
}
if (this.options.waitForProfile && !this.profileId) {
if (this.shouldQueue(payload)) {
this.queue.push(payload);
return Promise.resolve();
}
return this.api.fetch('/track', payload);
const result = await this.api.fetch<
TrackHandlerPayload,
{ deviceId: string; sessionId: string }
>('/track', payload);
this.deviceId = result?.deviceId;
const hadSession = !!this.sessionId;
this.sessionId = result?.sessionId;
// Flush queued items (e.g. replay chunks) when sessionId first arrives
if (!hadSession && this.sessionId) {
this.flush();
}
return result;
}
setGlobalProperties(properties: Record<string, unknown>) {
@@ -160,33 +186,44 @@ export class OpenPanel {
});
}
getDeviceId(): string {
return this.deviceId ?? '';
}
getSessionId(): string {
return this.sessionId ?? '';
}
async fetchDeviceId(): Promise<string> {
const result = await this.api.fetch<undefined, { deviceId: string }>(
'/track/device-id',
undefined,
{ method: 'GET', keepalive: false },
);
return result?.deviceId ?? '';
return Promise.resolve(this.deviceId ?? '');
}
clear() {
this.profileId = undefined;
// should we force a session end here?
this.deviceId = undefined;
this.sessionId = undefined;
}
flush() {
this.queue.forEach((item) => {
this.send({
...item,
// Not sure why ts-expect-error is needed here
// @ts-expect-error
payload: {
...item.payload,
profileId: item.payload.profileId ?? this.profileId,
},
});
});
this.queue = [];
const remaining: TrackHandlerPayload[] = [];
for (const item of this.queue) {
if (this.shouldQueue(item)) {
remaining.push(item);
continue;
}
const payload =
item.type === 'replay'
? item.payload
: {
...item.payload,
profileId:
'profileId' in item.payload
? (item.payload.profileId ?? this.profileId)
: this.profileId,
};
this.send({ ...item, payload } as TrackHandlerPayload);
}
this.queue = remaining;
}
log(...args: any[]) {

View File

@@ -11,6 +11,8 @@
},
"dependencies": {
"@openpanel/sdk": "workspace:1.0.4-local"
"@rrweb/types": "2.0.0-alpha.20",
"rrweb": "2.0.0-alpha.20"
},
"devDependencies": {
"@openpanel/tsconfig": "workspace:*",

View File

@@ -7,11 +7,36 @@ import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
export type * from '@openpanel/sdk';
export { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
export type SessionReplayOptions = {
enabled: boolean;
sampleRate?: number;
maskAllInputs?: boolean;
maskTextSelector?: string;
blockSelector?: string;
blockClass?: string;
ignoreSelector?: string;
flushIntervalMs?: number;
maxEventsPerChunk?: number;
maxPayloadBytes?: number;
/**
* URL to the replay recorder script.
* Only used when loading the SDK via a script tag (IIFE / op1.js).
* When using the npm package with a bundler this option is ignored
* because the bundler resolves the replay module from the package.
*/
scriptUrl?: string;
};
// Injected at build time only in the IIFE (tracker) build.
// In the library build this is `undefined`.
declare const __OPENPANEL_REPLAY_URL__: string | undefined;
export type OpenPanelOptions = OpenPanelBaseOptions & {
trackOutgoingLinks?: boolean;
trackScreenViews?: boolean;
trackAttributes?: boolean;
trackHashChanges?: boolean;
sessionReplay?: SessionReplayOptions;
};
function toCamelCase(str: string) {
@@ -66,6 +91,76 @@ export class OpenPanel extends OpenPanelBase {
if (this.options.trackAttributes) {
this.trackAttributes();
}
if (this.options.sessionReplay?.enabled) {
const sampleRate = this.options.sessionReplay.sampleRate ?? 1;
const sampled = Math.random() < sampleRate;
if (sampled) {
this.loadReplayModule().then((mod) => {
if (!mod) return;
mod.startReplayRecorder(this.options.sessionReplay!, (chunk) => {
this.send({
type: 'replay',
payload: {
...chunk,
},
});
});
});
}
}
}
}
/**
* Load the replay recorder module.
*
* - **IIFE build (op1.js)**: `__OPENPANEL_REPLAY_URL__` is replaced at
* build time with a CDN URL (e.g. `https://openpanel.dev/op1-replay.js`).
* The user can also override it via `sessionReplay.scriptUrl`.
* We load the IIFE replay script via a classic `<script>` tag which
* avoids CORS issues (dynamic `import(url)` uses `cors` mode).
* The IIFE exposes its exports on `window.__openpanel_replay`.
*
* - **Library build (npm)**: `__OPENPANEL_REPLAY_URL__` is `undefined`
* (never replaced). We use `import('./replay')` which the host app's
* bundler resolves and code-splits from the package source.
*/
private async loadReplayModule(): Promise<typeof import('./replay') | null> {
try {
// typeof check avoids a ReferenceError when the constant is not
// defined (library build). tsup replaces the constant with a
// string literal only in the IIFE build, so this branch is
// dead-code-eliminated in the library build.
if (typeof __OPENPANEL_REPLAY_URL__ !== 'undefined') {
// IIFE / script-tag context — load from CDN (or user override)
const url =
this.options.sessionReplay?.scriptUrl ?? __OPENPANEL_REPLAY_URL__;
// Already loaded (e.g. user included the script manually)
if ((window as any).__openpanel_replay) {
return (window as any).__openpanel_replay;
}
// Load via classic <script> tag — no CORS restrictions
return new Promise((resolve) => {
const script = document.createElement('script');
script.src = url;
script.onload = () => {
resolve((window as any).__openpanel_replay ?? null);
};
script.onerror = () => {
console.warn('[OpenPanel] Failed to load replay script from', url);
resolve(null);
};
document.head.appendChild(script);
});
}
// Library / bundler context — resolved by the bundler
return await import('./replay');
} catch (e) {
console.warn('[OpenPanel] Failed to load replay module', e);
return null;
}
}

View File

@@ -0,0 +1,2 @@
export { startReplayRecorder, stopReplayRecorder } from './recorder';
export type { ReplayChunkPayload, ReplayRecorderConfig } from './recorder';

View File

@@ -0,0 +1,131 @@
import type { eventWithTime } from 'rrweb';
import { record } from 'rrweb';
export type ReplayRecorderConfig = {
maskAllInputs?: boolean;
maskTextSelector?: string;
blockSelector?: string;
blockClass?: string;
ignoreSelector?: string;
flushIntervalMs?: number;
maxEventsPerChunk?: number;
maxPayloadBytes?: number;
};
export type ReplayChunkPayload = {
chunk_index: number;
events_count: number;
is_full_snapshot: boolean;
started_at: string;
ended_at: string;
payload: string;
};
let stopRecording: (() => void) | null = null;
export function startReplayRecorder(
config: ReplayRecorderConfig,
sendChunk: (payload: ReplayChunkPayload) => void,
): void {
if (typeof document === 'undefined' || typeof window === 'undefined') {
return;
}
const maxEventsPerChunk = config.maxEventsPerChunk ?? 200;
const flushIntervalMs = config.flushIntervalMs ?? 10_000;
const maxPayloadBytes = config.maxPayloadBytes ?? 1_048_576; // 1MB
let buffer: eventWithTime[] = [];
let chunkIndex = 0;
let flushTimer: ReturnType<typeof setInterval> | null = null;
function flush(isFullSnapshot: boolean): void {
if (buffer.length === 0) return;
const startedAt = buffer[0]!.timestamp;
const endedAt = buffer[buffer.length - 1]!.timestamp;
const payloadJson = JSON.stringify(buffer);
if (payloadJson.length > maxPayloadBytes) {
// If over size limit, split by taking only up to maxPayloadBytes (simplified: flush as-is and log or truncate)
// For MVP we still send; server will reject if over 1MB
}
sendChunk({
chunk_index: chunkIndex,
events_count: buffer.length,
is_full_snapshot: isFullSnapshot,
started_at: new Date(startedAt).toISOString(),
ended_at: new Date(endedAt).toISOString(),
payload: payloadJson,
});
chunkIndex += 1;
buffer = [];
}
function flushIfNeeded(isCheckout: boolean): void {
const isFullSnapshot =
isCheckout ||
buffer.some((e) => e.type === 2); /* EventType.FullSnapshot */
if (buffer.length >= maxEventsPerChunk) {
flush(isFullSnapshot);
} else if (isCheckout && buffer.length > 0) {
flush(true);
}
}
const stopFn = record({
emit(event: eventWithTime, isCheckout?: boolean) {
buffer.push(event);
flushIfNeeded(!!isCheckout);
},
checkoutEveryNms: flushIntervalMs,
maskAllInputs: config.maskAllInputs ?? true,
maskTextSelector: config.maskTextSelector ?? '[data-openpanel-replay-mask]',
blockSelector: config.blockSelector ?? '[data-openpanel-replay-block]',
blockClass: config.blockClass,
ignoreSelector: config.ignoreSelector,
});
flushTimer = setInterval(() => {
if (buffer.length > 0) {
const hasFullSnapshot = buffer.some((e) => e.type === 2);
flush(hasFullSnapshot);
}
}, flushIntervalMs);
function onVisibilityChange(): void {
if (document.visibilityState === 'hidden' && buffer.length > 0) {
const hasFullSnapshot = buffer.some((e) => e.type === 2);
flush(hasFullSnapshot);
}
}
function onPageHide(): void {
if (buffer.length > 0) {
const hasFullSnapshot = buffer.some((e) => e.type === 2);
flush(hasFullSnapshot);
}
}
document.addEventListener('visibilitychange', onVisibilityChange);
window.addEventListener('pagehide', onPageHide);
stopRecording = () => {
if (flushTimer) {
clearInterval(flushTimer);
flushTimer = null;
}
document.removeEventListener('visibilitychange', onVisibilityChange);
window.removeEventListener('pagehide', onPageHide);
stopFn?.();
stopRecording = null;
};
}
export function stopReplayRecorder(): void {
if (stopRecording) {
stopRecording();
}
}

View File

@@ -1,11 +1,47 @@
import { defineConfig } from 'tsup';
export default defineConfig({
entry: ['index.ts', 'src/tracker.ts'],
format: ['cjs', 'esm', 'iife'],
dts: true,
splitting: false,
sourcemap: false,
clean: true,
minify: true,
});
export default defineConfig([
// Library build (npm package) — cjs + esm + dts
// Dynamic import('./replay') is preserved; the host app's bundler
// will code-split it into a separate chunk automatically.
{
entry: ['index.ts'],
format: ['cjs', 'esm'],
dts: true,
splitting: false,
sourcemap: false,
clean: true,
minify: true,
},
// IIFE build (script tag: op1.js)
// __OPENPANEL_REPLAY_URL__ is injected at build time so the IIFE
// knows to load the replay module from the CDN instead of a
// relative import (which doesn't work in a standalone script).
{
entry: { 'src/tracker': 'src/tracker.ts' },
format: ['iife'],
splitting: false,
sourcemap: false,
minify: true,
define: {
__OPENPANEL_REPLAY_URL__: JSON.stringify(
'https://openpanel.dev/op1-replay.js',
),
},
},
// Replay module — built as both ESM (npm) and IIFE (CDN).
// ESM → consumed by the host-app's bundler via `import('./replay')`.
// IIFE → loaded at runtime via a classic <script> tag (no CORS issues).
// Exposes `window.__openpanel_replay`.
// rrweb must be bundled in (noExternal) because browsers can't resolve
// bare specifiers like "rrweb" from a standalone ES module / script.
{
entry: { 'src/replay': 'src/replay/index.ts' },
format: ['esm', 'iife'],
globalName: '__openpanel_replay',
splitting: false,
sourcemap: false,
minify: true,
noExternal: ['rrweb', '@rrweb/types'],
},
]);

View File

@@ -1,6 +1,10 @@
import { z } from 'zod';
import { getSessionList, sessionService } from '@openpanel/db';
import {
getSessionList,
getSessionReplayEvents,
sessionService,
} from '@openpanel/db';
import { zChartEventFilter } from '@openpanel/validation';
import { createTRPCRouter, protectedProcedure } from '../trpc';
@@ -61,4 +65,10 @@ export const sessionRouter = createTRPCRouter({
.query(async ({ input: { sessionId, projectId } }) => {
return sessionService.byId(sessionId, projectId);
}),
replay: protectedProcedure
.input(z.object({ sessionId: z.string(), projectId: z.string() }))
.query(async ({ input: { sessionId, projectId } }) => {
return getSessionReplayEvents(sessionId, projectId);
}),
});

View File

@@ -63,6 +63,15 @@ export const zAliasPayload = z.object({
alias: z.string().min(1),
});
export const zReplayPayload = z.object({
chunk_index: z.number().int().min(0).max(65535),
events_count: z.number().int().min(1),
is_full_snapshot: z.boolean(),
started_at: z.string(),
ended_at: z.string(),
payload: z.string().max(1_048_576), // 1MB max
});
export const zTrackHandlerPayload = z.discriminatedUnion('type', [
z.object({
type: z.literal('track'),
@@ -84,6 +93,10 @@ export const zTrackHandlerPayload = z.discriminatedUnion('type', [
type: z.literal('alias'),
payload: zAliasPayload,
}),
z.object({
type: z.literal('replay'),
payload: zReplayPayload,
}),
]);
export type ITrackPayload = z.infer<typeof zTrackPayload>;
@@ -91,6 +104,7 @@ export type IIdentifyPayload = z.infer<typeof zIdentifyPayload>;
export type IIncrementPayload = z.infer<typeof zIncrementPayload>;
export type IDecrementPayload = z.infer<typeof zDecrementPayload>;
export type IAliasPayload = z.infer<typeof zAliasPayload>;
export type IReplayPayload = z.infer<typeof zReplayPayload>;
export type ITrackHandlerPayload = z.infer<typeof zTrackHandlerPayload>;
// Deprecated types for beta version of the SDKs