zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: session replay

Browse Source 
* wip

* wip

* wip

* wip

* final fixes

* comments

* fix
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-02-26 14:09:53 +01:00
committed by GitHub
parent 38d9b65ec8
commit aa81bbfe77
67 changed files with 3059 additions and 556 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
apps/api/src/controllers/event.controller.ts
View File

@@ -1,26 +1,25 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import { generateId, slug } from '@openpanel/common';
import { parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { getGeoLocation } from '@openpanel/geo';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import type { DeprecatedPostEventPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { getStringHeaders, getTimestamp } from './track.controller';
import { getDeviceId } from '@/utils/ids';
export async function postEvent(
request: FastifyRequest<{
Body: DeprecatedPostEventPayload;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const { timestamp, isTimestampFromThePast } = getTimestamp(
request.timestamp,
request.body,
request.body
);
const ip = request.clientIp;
const ua = request.headers['user-agent'];
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
const projectId = request.client?.projectId;
const headers = getStringHeaders(request.headers);
@@ -30,34 +29,22 @@ export async function postEvent(
}
const [salts, geo] = await Promise.all([getSalts(), getGeoLocation(ip)]);
const currentDeviceId = ua
? generateDeviceId({
salt: salts.current,
origin: projectId,
ip,
ua,
})
: '';
const previousDeviceId = ua
? generateDeviceId({
salt: salts.previous,
origin: projectId,
ip,
ua,
})
: '';
const { deviceId, sessionId } = await getDeviceId({
projectId,
ip,
ua,
salts,
});
const uaInfo = parseUserAgent(ua, request.body?.properties);
const groupId = uaInfo.isServer
? request.body?.profileId
? `${projectId}:${request.body?.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
? `${projectId}:${request.body?.profileId ?? generateId()}`
: deviceId;
const jobId = [
slug(request.body.name),
timestamp,
projectId,
currentDeviceId,
deviceId,
groupId,
]
.filter(Boolean)
@@ -74,8 +61,10 @@ export async function postEvent(
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
currentDeviceId: '',
previousDeviceId: '',
deviceId,
sessionId: sessionId ?? '',
},
groupId,
jobId,

190
apps/api/src/controllers/track.controller.ts
View File

@@ -1,22 +1,27 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, pathOr, pick } from 'ramda';
import { HttpError } from '@/utils/errors';
import { generateId, slug } from '@openpanel/common';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import {
getProfileById,
getSalts,
replayBuffer,
upsertProfile,
} from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import {
type IDecrementPayload,
type IIdentifyPayload,
type IIncrementPayload,
type IReplayPayload,
type ITrackHandlerPayload,
type ITrackPayload,
zTrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, pathOr, pick } from 'ramda';
import { HttpError } from '@/utils/errors';
import { getDeviceId } from '@/utils/ids';
export function getStringHeaders(headers: FastifyRequest['headers']) {
return Object.entries(
@@ -28,14 +33,14 @@ export function getStringHeaders(headers: FastifyRequest['headers']) {
'openpanel-client-id',
'request-id',
],
headers,
),
headers
)
).reduce(
(acc, [key, value]) => ({
...acc,
[key]: value ? String(value) : undefined,
}),
{},
{}
);
}
@@ -45,14 +50,15 @@ function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {
| IIdentifyPayload
| undefined;
return (
identity ||
(body.payload.profileId
? {
profileId: String(body.payload.profileId),
}
: undefined)
);
if (identity) {
return identity;
}
return body.payload.profileId
? {
profileId: String(body.payload.profileId),
}
: undefined;
}
return undefined;
@@ -60,7 +66,7 @@ function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {
export function getTimestamp(
timestamp: FastifyRequest['timestamp'],
payload: ITrackHandlerPayload['payload'],
payload: ITrackHandlerPayload['payload']
) {
const safeTimestamp = timestamp || Date.now();
const userDefinedTimestamp =
@@ -104,8 +110,8 @@ interface TrackContext {
headers: Record<string, string | undefined>;
timestamp: { value: number; isFromPast: boolean };
identity?: IIdentifyPayload;
currentDeviceId?: string;
previousDeviceId?: string;
deviceId: string;
sessionId: string;
geo: GeoLocation;
}
@@ -113,7 +119,7 @@ async function buildContext(
request: FastifyRequest<{
Body: ITrackHandlerPayload;
}>,
validatedBody: ITrackHandlerPayload,
validatedBody: ITrackHandlerPayload
): Promise<TrackContext> {
const projectId = request.client?.projectId;
if (!projectId) {
@@ -128,49 +134,27 @@ async function buildContext(
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
const headers = getStringHeaders(request.headers);
const identity = getIdentity(validatedBody);
const profileId = identity?.profileId;
// We might get a profileId from the alias table
// If we do, we should use that instead of the one from the payload
if (profileId && validatedBody.type === 'track') {
validatedBody.payload.profileId = profileId;
}
// Get geo location (needed for track and identify)
const geo = await getGeoLocation(ip);
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
// Generate device IDs if needed (for track)
let currentDeviceId: string | undefined;
let previousDeviceId: string | undefined;
if (validatedBody.type === 'track') {
const overrideDeviceId =
typeof validatedBody.payload.properties?.__deviceId === 'string'
? validatedBody.payload.properties.__deviceId
: undefined;
const salts = await getSalts();
currentDeviceId =
overrideDeviceId ||
(ua
? generateDeviceId({
salt: salts.current,
origin: projectId,
ip,
ua,
})
: '');
previousDeviceId = ua
? generateDeviceId({
salt: salts.previous,
origin: projectId,
ip,
ua,
})
: '';
}
const { deviceId, sessionId } = await getDeviceId({
projectId,
ip,
ua,
salts,
overrideDeviceId:
validatedBody.type === 'track' &&
typeof validatedBody.payload?.properties?.__deviceId === 'string'
? validatedBody.payload?.properties.__deviceId
: undefined,
});
return {
projectId,
@@ -182,46 +166,35 @@ async function buildContext(
isFromPast: timestamp.isTimestampFromThePast,
},
identity,
currentDeviceId,
previousDeviceId,
deviceId,
sessionId,
geo,
};
}
async function handleTrack(
payload: ITrackPayload,
context: TrackContext,
context: TrackContext
): Promise<void> {
const {
projectId,
currentDeviceId,
previousDeviceId,
geo,
headers,
timestamp,
} = context;
if (!currentDeviceId || !previousDeviceId) {
throw new HttpError('Device ID generation failed', { status: 500 });
}
const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
const groupId = uaInfo.isServer
? payload.profileId
? `${projectId}:${payload.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
: deviceId;
const jobId = [
slug(payload.name),
timestamp.value,
projectId,
currentDeviceId,
deviceId,
groupId,
]
.filter(Boolean)
.join('-');
const promises = [];
const promises: Promise<unknown>[] = [];
// If we have more than one property in the identity object, we should identify the user
// Otherwise its only a profileId and we should not identify the user
@@ -242,12 +215,14 @@ async function handleTrack(
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
deviceId,
sessionId,
currentDeviceId: '', // TODO: Remove
previousDeviceId: '', // TODO: Remove
},
groupId,
jobId,
}),
})
);
await Promise.all(promises);
@@ -255,7 +230,7 @@ async function handleTrack(
async function handleIdentify(
payload: IIdentifyPayload,
context: TrackContext,
context: TrackContext
): Promise<void> {
const { projectId, geo, ua } = context;
const uaInfo = parseUserAgent(ua, payload.properties);
@@ -285,7 +260,7 @@ async function handleIdentify(
async function adjustProfileProperty(
payload: IIncrementPayload | IDecrementPayload,
projectId: string,
direction: 1 | -1,
direction: 1 | -1
): Promise<void> {
const { profileId, property, value } = payload;
const profile = await getProfileById(profileId, projectId);
@@ -295,7 +270,7 @@ async function adjustProfileProperty(
const parsed = Number.parseInt(
pathOr<string>('0', property.split('.'), profile.properties),
10,
10
);
if (Number.isNaN(parsed)) {
@@ -305,7 +280,7 @@ async function adjustProfileProperty(
profile.properties = assocPath(
property.split('.'),
parsed + direction * (value || 1),
profile.properties,
profile.properties
);
await upsertProfile({
@@ -318,23 +293,44 @@ async function adjustProfileProperty(
async function handleIncrement(
payload: IIncrementPayload,
context: TrackContext,
context: TrackContext
): Promise<void> {
await adjustProfileProperty(payload, context.projectId, 1);
}
async function handleDecrement(
payload: IDecrementPayload,
context: TrackContext,
context: TrackContext
): Promise<void> {
await adjustProfileProperty(payload, context.projectId, -1);
}
async function handleReplay(
payload: IReplayPayload,
context: TrackContext
): Promise<void> {
if (!context.sessionId) {
throw new HttpError('Session ID is required for replay', { status: 400 });
}
const row = {
project_id: context.projectId,
session_id: context.sessionId,
chunk_index: payload.chunk_index,
started_at: payload.started_at,
ended_at: payload.ended_at,
events_count: payload.events_count,
is_full_snapshot: payload.is_full_snapshot,
payload: payload.payload,
};
await replayBuffer.add(row);
}
export async function handler(
request: FastifyRequest<{
Body: ITrackHandlerPayload;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
// Validate request body with Zod
const validationResult = zTrackHandlerPayload.safeParse(request.body);
@@ -375,6 +371,9 @@ export async function handler(
case 'decrement':
await handleDecrement(validatedBody.payload, context);
break;
case 'replay':
await handleReplay(validatedBody.payload, context);
break;
default:
return reply.status(400).send({
status: 400,
@@ -383,12 +382,15 @@ export async function handler(
});
}
reply.status(200).send();
reply.status(200).send({
deviceId: context.deviceId,
sessionId: context.sessionId,
});
}
export async function fetchDeviceId(
request: FastifyRequest,
reply: FastifyReply,
reply: FastifyReply
) {
const salts = await getSalts();
const projectId = request.client?.projectId;
@@ -421,20 +423,31 @@ export async function fetchDeviceId(
try {
const multi = getRedisCache().multi();
multi.exists(`bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`);
multi.exists(`bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`);
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`,
'data'
);
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`,
'data'
);
const res = await multi.exec();
if (res?.[0]?.[1]) {
const data = JSON.parse(res?.[0]?.[1] as string);
const sessionId = data.payload.sessionId;
return reply.status(200).send({
deviceId: currentDeviceId,
sessionId,
message: 'current session exists for this device id',
});
}
if (res?.[1]?.[1]) {
const data = JSON.parse(res?.[1]?.[1] as string);
const sessionId = data.payload.sessionId;
return reply.status(200).send({
deviceId: previousDeviceId,
sessionId,
message: 'previous session exists for this device id',
});
}
@@ -444,6 +457,7 @@ export async function fetchDeviceId(
return reply.status(200).send({
deviceId: currentDeviceId,
sessionId: '',
message: 'No session exists for this device id',
});
}

7
apps/api/src/hooks/duplicate.hook.ts
View File

@@ -1,20 +1,21 @@
import { isDuplicatedEvent } from '@/utils/deduplicate';
import type {
DeprecatedPostEventPayload,
ITrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { isDuplicatedEvent } from '@/utils/deduplicate';
export async function duplicateHook(
req: FastifyRequest<{
Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
}>,
reply: FastifyReply,
reply: FastifyReply
) {
const ip = req.clientIp;
const origin = req.headers.origin;
const clientId = req.headers['openpanel-client-id'];
const shouldCheck = ip && origin && clientId;
const isReplay = 'type' in req.body && req.body.type === 'replay';
const shouldCheck = ip && origin && clientId && !isReplay;
const isDuplicate = shouldCheck
? await isDuplicatedEvent({

8
apps/api/src/hooks/request-logging.hook.ts
View File

@@ -1,4 +1,3 @@
import { DEFAULT_IP_HEADER_ORDER } from '@openpanel/common';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { path, pick } from 'ramda';
@@ -6,7 +5,7 @@ const ignoreLog = ['/healthcheck', '/healthz', '/metrics', '/misc'];
const ignoreMethods = ['OPTIONS'];
const getTrpcInput = (
request: FastifyRequest,
request: FastifyRequest
): Record<string, unknown> | undefined => {
const input = path<any>(['query', 'input'], request);
try {
@@ -18,7 +17,7 @@ const getTrpcInput = (
export async function requestLoggingHook(
request: FastifyRequest,
reply: FastifyReply,
reply: FastifyReply
) {
if (ignoreMethods.includes(request.method)) {
return;
@@ -40,9 +39,8 @@ export async function requestLoggingHook(
elapsed: reply.elapsedTime,
headers: pick(
['openpanel-client-id', 'openpanel-sdk-name', 'openpanel-sdk-version'],
request.headers,
request.headers
),
body: request.body,
});
}
}

41
apps/api/src/index.ts
View File

@@ -3,12 +3,12 @@ process.env.TZ = 'UTC';
import compress from '@fastify/compress';
import cookie from '@fastify/cookie';
import cors, { type FastifyCorsOptions } from '@fastify/cors';
import type { FastifyTRPCPluginOptions } from '@trpc/server/adapters/fastify';
import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
import type { FastifyBaseLogger, FastifyRequest } from 'fastify';
import Fastify from 'fastify';
import metricsPlugin from 'fastify-metrics';
import {
decodeSessionToken,
EMPTY_SESSION,
type SessionValidationResult,
validateSessionToken,
} from '@openpanel/auth';
import { generateId } from '@openpanel/common';
import {
type IServiceClientWithProject,
@@ -17,13 +17,11 @@ import {
import { getRedisPub } from '@openpanel/redis';
import type { AppRouter } from '@openpanel/trpc';
import { appRouter, createContext } from '@openpanel/trpc';
import {
EMPTY_SESSION,
type SessionValidationResult,
decodeSessionToken,
validateSessionToken,
} from '@openpanel/auth';
import type { FastifyTRPCPluginOptions } from '@trpc/server/adapters/fastify';
import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
import type { FastifyBaseLogger, FastifyRequest } from 'fastify';
import Fastify from 'fastify';
import metricsPlugin from 'fastify-metrics';
import sourceMapSupport from 'source-map-support';
import {
healthcheck,
@@ -72,7 +70,7 @@ const startServer = async () => {
try {
const fastify = Fastify({
maxParamLength: 15_000,
bodyLimit: 1048576 * 500, // 500MB
bodyLimit: 1_048_576 * 500, // 500MB
loggerInstance: logger as unknown as FastifyBaseLogger,
disableRequestLogging: true,
genReqId: (req) =>
@@ -84,7 +82,7 @@ const startServer = async () => {
fastify.register(cors, () => {
return (
req: FastifyRequest,
callback: (error: Error | null, options: FastifyCorsOptions) => void,
callback: (error: Error | null, options: FastifyCorsOptions) => void
) => {
// TODO: set prefix on dashboard routes
const corsPaths = [
@@ -97,7 +95,7 @@ const startServer = async () => {
];
const isPrivatePath = corsPaths.some((path) =>
req.url.startsWith(path),
req.url.startsWith(path)
);
if (isPrivatePath) {
@@ -118,6 +116,7 @@ const startServer = async () => {
return callback(null, {
origin: '*',
maxAge: 86_400 * 7, // cache preflight for 7 days
});
};
});
@@ -149,7 +148,7 @@ const startServer = async () => {
try {
const sessionId = decodeSessionToken(req.cookies?.session);
const session = await runWithAlsSession(sessionId, () =>
validateSessionToken(req.cookies.session),
validateSessionToken(req.cookies.session)
);
req.session = session;
} catch (e) {
@@ -158,7 +157,7 @@ const startServer = async () => {
} else if (process.env.DEMO_USER_ID) {
try {
const session = await runWithAlsSession('1', () =>
validateSessionToken(null),
validateSessionToken(null)
);
req.session = session;
} catch (e) {
@@ -173,7 +172,7 @@ const startServer = async () => {
prefix: '/trpc',
trpcOptions: {
router: appRouter,
createContext: createContext,
createContext,
onError(ctx) {
if (
ctx.error.code === 'UNAUTHORIZED' &&
@@ -217,7 +216,7 @@ const startServer = async () => {
reply.send({
status: 'ok',
message: 'Successfully running OpenPanel.dev API',
}),
})
);
});
@@ -274,7 +273,7 @@ const startServer = async () => {
} catch (error) {
logger.warn('Failed to set redis notify-keyspace-events', error);
logger.warn(
'If you use a managed Redis service, you may need to set this manually.',
'If you use a managed Redis service, you may need to set this manually.'
);
logger.warn('Otherwise some functions may not work as expected.');
}

1
apps/api/src/routes/track.router.ts
View File

@@ -26,6 +26,7 @@ const trackRouter: FastifyPluginCallback = async (fastify) => {
type: 'object',
properties: {
deviceId: { type: 'string' },
sessionId: { type: 'string' },
message: { type: 'string', optional: true },
},
},

158
apps/api/src/utils/ids.ts Normal file
View File

@@ -0,0 +1,158 @@
import crypto from 'node:crypto';
import { generateDeviceId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json';
import { getRedisCache } from '@openpanel/redis';
export async function getDeviceId({
projectId,
ip,
ua,
salts,
overrideDeviceId,
}: {
projectId: string;
ip: string;
ua: string | undefined;
salts: { current: string; previous: string };
overrideDeviceId?: string;
}) {
if (overrideDeviceId) {
return { deviceId: overrideDeviceId, sessionId: '' };
}
if (!ua) {
return { deviceId: '', sessionId: '' };
}
const currentDeviceId = generateDeviceId({
salt: salts.current,
origin: projectId,
ip,
ua,
});
const previousDeviceId = generateDeviceId({
salt: salts.previous,
origin: projectId,
ip,
ua,
});
return await getDeviceIdFromSession({
projectId,
currentDeviceId,
previousDeviceId,
});
}
async function getDeviceIdFromSession({
projectId,
currentDeviceId,
previousDeviceId,
}: {
projectId: string;
currentDeviceId: string;
previousDeviceId: string;
}) {
try {
const multi = getRedisCache().multi();
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`,
'data'
);
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`,
'data'
);
const res = await multi.exec();
if (res?.[0]?.[1]) {
const data = getSafeJson<{ payload: { sessionId: string } }>(
(res?.[0]?.[1] as string) ?? ''
);
if (data) {
const sessionId = data.payload.sessionId;
return { deviceId: currentDeviceId, sessionId };
}
}
if (res?.[1]?.[1]) {
const data = getSafeJson<{ payload: { sessionId: string } }>(
(res?.[1]?.[1] as string) ?? ''
);
if (data) {
const sessionId = data.payload.sessionId;
return { deviceId: previousDeviceId, sessionId };
}
}
} catch (error) {
console.error('Error getting session end GET /track/device-id', error);
}
return {
deviceId: currentDeviceId,
sessionId: getSessionId({
projectId,
deviceId: currentDeviceId,
graceMs: 5 * 1000,
windowMs: 1000 * 60 * 30,
}),
};
}
/**
* Deterministic session id for (projectId, deviceId) within a time window,
* with a grace period at the *start* of each window to avoid boundary splits.
*
* - windowMs: 30 minutes by default
* - graceMs: 1 minute by default (events in first minute of a bucket map to previous bucket)
* - Output: base64url, 128-bit (16 bytes) truncated from SHA-256
*/
function getSessionId(params: {
projectId: string;
deviceId: string;
eventMs?: number; // use event timestamp; defaults to Date.now()
windowMs?: number; // default 5 min
graceMs?: number; // default 1 min
bytes?: number; // default 16 (128-bit). You can set 24 or 32 for longer ids.
}): string {
const {
projectId,
deviceId,
eventMs = Date.now(),
windowMs = 5 * 60 * 1000,
graceMs = 60 * 1000,
bytes = 16,
} = params;
if (!projectId) {
throw new Error('projectId is required');
}
if (!deviceId) {
throw new Error('deviceId is required');
}
if (windowMs <= 0) {
throw new Error('windowMs must be > 0');
}
if (graceMs < 0 || graceMs >= windowMs) {
throw new Error('graceMs must be >= 0 and < windowMs');
}
if (bytes < 8 || bytes > 32) {
throw new Error('bytes must be between 8 and 32');
}
const bucket = Math.floor(eventMs / windowMs);
const offset = eventMs - bucket * windowMs;
// Grace at the start of the bucket: stick to the previous bucket.
const chosenBucket = offset < graceMs ? bucket - 1 : bucket;
const input = `sess:v1:${projectId}:${deviceId}:${chosenBucket}`;
const digest = crypto.createHash('sha256').update(input).digest();
const truncated = digest.subarray(0, bytes);
// base64url
return truncated
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/g, '');
}