This commit is contained in:
Carl-Gerhard Lindesvärd
2026-02-10 22:19:10 +00:00
parent 551927af06
commit 47adf46625
6 changed files with 251 additions and 135 deletions

View File

@@ -3,11 +3,7 @@ import { assocPath, pathOr, pick } from 'ramda';
import { HttpError } from '@/utils/errors';
import { generateId, slug } from '@openpanel/common';
import {
generateDeviceId,
generateSecureId,
parseUserAgent,
} from '@openpanel/common/server';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import {
TABLE_NAMES,
ch,
@@ -20,6 +16,7 @@ import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import { getDeviceId } from '@/utils/ids';
import {
type IDecrementPayload,
type IIdentifyPayload,
@@ -30,84 +27,6 @@ import {
zTrackHandlerPayload,
} from '@openpanel/validation';
async function getDeviceId({
projectId,
ip,
ua,
salts,
overrideDeviceId,
}: {
projectId: string;
ip: string;
ua: string | undefined;
salts: { current: string; previous: string };
overrideDeviceId?: string;
}) {
if (overrideDeviceId) {
return { deviceId: overrideDeviceId, sessionId: undefined };
}
if (!ua) {
return { deviceId: '', sessionId: undefined };
}
const currentDeviceId = generateDeviceId({
salt: salts.current,
origin: projectId,
ip,
ua,
});
const previousDeviceId = generateDeviceId({
salt: salts.previous,
origin: projectId,
ip,
ua,
});
return await getDeviceIdFromSession({
projectId,
currentDeviceId,
previousDeviceId,
});
}
async function getDeviceIdFromSession({
projectId,
currentDeviceId,
previousDeviceId,
}: {
projectId: string;
currentDeviceId: string;
previousDeviceId: string;
}) {
try {
const multi = getRedisCache().multi();
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`,
'data',
);
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`,
'data',
);
const res = await multi.exec();
if (res?.[0]?.[1]) {
const data = JSON.parse(res?.[0]?.[1] as string);
const sessionId = data.payload.sessionId;
return { deviceId: currentDeviceId, sessionId };
}
if (res?.[1]?.[1]) {
const data = JSON.parse(res?.[1]?.[1] as string);
const sessionId = data.payload.sessionId;
return { deviceId: previousDeviceId, sessionId };
}
} catch (error) {
console.error('Error getting session end GET /track/device-id', error);
}
return { deviceId: currentDeviceId, sessionId: generateSecureId('se') };
}
export function getStringHeaders(headers: FastifyRequest['headers']) {
return Object.entries(
pick(

141
apps/api/src/utils/ids.ts Normal file
View File

@@ -0,0 +1,141 @@
import crypto from 'node:crypto';
import { generateDeviceId } from '@openpanel/common/server';
import { getRedisCache } from '@openpanel/redis';
export async function getDeviceId({
projectId,
ip,
ua,
salts,
overrideDeviceId,
}: {
projectId: string;
ip: string;
ua: string | undefined;
salts: { current: string; previous: string };
overrideDeviceId?: string;
}) {
if (overrideDeviceId) {
return { deviceId: overrideDeviceId, sessionId: undefined };
}
if (!ua) {
return { deviceId: '', sessionId: undefined };
}
const currentDeviceId = generateDeviceId({
salt: salts.current,
origin: projectId,
ip,
ua,
});
const previousDeviceId = generateDeviceId({
salt: salts.previous,
origin: projectId,
ip,
ua,
});
return await getDeviceIdFromSession({
projectId,
currentDeviceId,
previousDeviceId,
});
}
async function getDeviceIdFromSession({
projectId,
currentDeviceId,
previousDeviceId,
}: {
projectId: string;
currentDeviceId: string;
previousDeviceId: string;
}) {
try {
const multi = getRedisCache().multi();
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`,
'data',
);
multi.hget(
`bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`,
'data',
);
const res = await multi.exec();
if (res?.[0]?.[1]) {
const data = JSON.parse(res?.[0]?.[1] as string);
const sessionId = data.payload.sessionId;
return { deviceId: currentDeviceId, sessionId };
}
if (res?.[1]?.[1]) {
const data = JSON.parse(res?.[1]?.[1] as string);
const sessionId = data.payload.sessionId;
return { deviceId: previousDeviceId, sessionId };
}
} catch (error) {
console.error('Error getting session end GET /track/device-id', error);
}
return {
deviceId: currentDeviceId,
sessionId: getSessionId({
projectId,
deviceId: currentDeviceId,
graceMs: 5 * 1000,
windowMs: 1000 * 60 * 30,
}),
};
}
/**
* Deterministic session id for (projectId, deviceId) within a time window,
* with a grace period at the *start* of each window to avoid boundary splits.
*
* - windowMs: 5 minutes by default
* - graceMs: 1 minute by default (events in first minute of a bucket map to previous bucket)
* - Output: base64url, 128-bit (16 bytes) truncated from SHA-256
*/
function getSessionId(params: {
projectId: string;
deviceId: string;
eventMs?: number; // use event timestamp; defaults to Date.now()
windowMs?: number; // default 5 min
graceMs?: number; // default 1 min
bytes?: number; // default 16 (128-bit). You can set 24 or 32 for longer ids.
}): string {
const {
projectId,
deviceId,
eventMs = Date.now(),
windowMs = 5 * 60 * 1000,
graceMs = 60 * 1000,
bytes = 16,
} = params;
if (!projectId) throw new Error('projectId is required');
if (!deviceId) throw new Error('deviceId is required');
if (windowMs <= 0) throw new Error('windowMs must be > 0');
if (graceMs < 0 || graceMs >= windowMs)
throw new Error('graceMs must be >= 0 and < windowMs');
if (bytes < 8 || bytes > 32)
throw new Error('bytes must be between 8 and 32');
const bucket = Math.floor(eventMs / windowMs);
const offset = eventMs - bucket * windowMs;
// Grace at the start of the bucket: stick to the previous bucket.
const chosenBucket = offset < graceMs ? bucket - 1 : bucket;
const input = `sess:v1:${projectId}:${deviceId}:${chosenBucket}`;
const digest = crypto.createHash('sha256').update(input).digest();
const truncated = digest.subarray(0, bytes);
// base64url
return truncated
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/g, '');
}

View File

@@ -194,13 +194,11 @@ export async function getSessionList({
const dateIntervalInDays =
organization?.subscriptionPeriodEventsLimit &&
organization?.subscriptionPeriodEventsLimit > 1_000_000
? 1
? 2
: 360;
if (cursor) {
const cAt = sqlstring.escape(cursor.createdAt);
// TODO: remove id from cursor
const cId = sqlstring.escape(cursor.id);
sb.where.cursor = `created_at < toDateTime64(${cAt}, 3)`;
sb.where.cursorWindow = `created_at >= toDateTime64(${cAt}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
sb.orderBy.created_at = 'created_at DESC';
@@ -330,24 +328,25 @@ export async function getSessionReplayEvents(
projectId: string,
): Promise<{ events: unknown[] }> {
const chunks = await clix(ch)
.select<{ chunk_index: number; payload: string }>(['chunk_index', 'payload'])
.select<{ chunk_index: number; payload: string }>([
'chunk_index',
'payload',
])
.from(TABLE_NAMES.session_replay_chunks)
.where('session_id', '=', sessionId)
.where('project_id', '=', projectId)
.orderBy('chunk_index', 'ASC')
.execute();
const allEvents = chunks.flatMap((chunk) =>
JSON.parse(chunk.payload) as unknown[],
const allEvents = chunks.flatMap(
(chunk) => JSON.parse(chunk.payload) as unknown[],
);
// rrweb event types: 2 = FullSnapshot, 4 = Meta
// Incremental snapshots (type 3) before the first FullSnapshot are orphaned
// and cause the player to fast-forward through empty time. Strip them but
// keep Meta events (type 4) since rrweb needs them for viewport dimensions.
const firstFullSnapshotIdx = allEvents.findIndex(
(e: any) => e.type === 2,
);
const firstFullSnapshotIdx = allEvents.findIndex((e: any) => e.type === 2);
let events = allEvents;
if (firstFullSnapshotIdx > 0) {

View File

@@ -10,7 +10,7 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@openpanel/sdk": "workspace:1.0.4-local"
"@openpanel/sdk": "workspace:1.0.4-local",
"@rrweb/types": "2.0.0-alpha.20",
"rrweb": "2.0.0-alpha.20"
},
@@ -20,4 +20,4 @@
"tsup": "^7.2.0",
"typescript": "catalog:"
}
}
}

98
pnpm-lock.yaml generated
View File

@@ -774,6 +774,9 @@ importers:
remark-rehype:
specifier: ^11.1.2
version: 11.1.2
rrweb-player:
specifier: 2.0.0-alpha.20
version: 2.0.0-alpha.20
short-unique-id:
specifier: ^5.0.3
version: 5.0.3
@@ -1629,6 +1632,12 @@ importers:
'@openpanel/sdk':
specifier: workspace:1.0.4-local
version: link:../sdk
'@rrweb/types':
specifier: 2.0.0-alpha.20
version: 2.0.0-alpha.20
rrweb:
specifier: 2.0.0-alpha.20
version: 2.0.0-alpha.20
devDependencies:
'@openpanel/tsconfig':
specifier: workspace:*
@@ -8705,6 +8714,18 @@ packages:
cpu: [x64]
os: [win32]
'@rrweb/packer@2.0.0-alpha.20':
resolution: {integrity: sha512-GsByg2olGZ2n3To6keFG604QAboipuXZvjYxO2wITSwARBf/sZdy6cbUEjF0RS+QnuTM5GaVXeQapNMLmpKbrA==}
'@rrweb/replay@2.0.0-alpha.20':
resolution: {integrity: sha512-VodsLb+C2bYNNVbb0U14tKLa9ctzUxYIlt9VnxPATWvfyXHLTku8BhRWptuW/iIjVjmG49LBoR1ilxw/HMiJ1w==}
'@rrweb/types@2.0.0-alpha.20':
resolution: {integrity: sha512-RbnDgKxA/odwB1R4gF7eUUj+rdSrq6ROQJsnMw7MIsGzlbSYvJeZN8YY4XqU0G6sKJvXI6bSzk7w/G94jNwzhw==}
'@rrweb/utils@2.0.0-alpha.20':
resolution: {integrity: sha512-MTQOmhPRe39C0fYaCnnVYOufQsyGzwNXpUStKiyFSfGLUJrzuwhbRoUAKR5w6W2j5XuA0bIz3ZDIBztkquOhLw==}
'@segment/loosely-validate-event@2.0.0':
resolution: {integrity: sha512-ZMCSfztDBqwotkl848ODgVcAmN4OItEWDCkshcKz0/W6gGSQayuuCtWV/MlodFivAZD793d6UgANd6wCXUfrIw==}
@@ -9656,6 +9677,9 @@ packages:
'@tsconfig/node18@1.0.3':
resolution: {integrity: sha512-RbwvSJQsuN9TB04AQbGULYfOGE/RnSFk/FLQ5b0NmDf5Kx2q/lABZbHQPKCO1vZ6Fiwkplu+yb9pGdLy1iGseQ==}
'@tsconfig/svelte@1.0.13':
resolution: {integrity: sha512-5lYJP45Xllo4yE/RUBccBT32eBlRDbqN8r1/MIvQbKxW3aFqaYPCNgm8D5V20X4ShHcwvYWNlKg3liDh1MlBoA==}
'@turf/boolean-point-in-polygon@6.5.0':
resolution: {integrity: sha512-DtSuVFB26SI+hj0SjrvXowGTUCHlgevPAIsukssW6BG5MlNSBQAo70wpICBNJL6RjukXg8d2eXaAWuD/CqL00A==}
@@ -9722,6 +9746,9 @@ packages:
'@types/cors@2.8.17':
resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==}
'@types/css-font-loading-module@0.0.7':
resolution: {integrity: sha512-nl09VhutdjINdWyXxHWN/w9zlNCfr60JUqJbd24YXUuCwgeL0TpFSdElCwb6cxfB6ybE19Gjj4g0jsgkXxKv1Q==}
'@types/d3-array@3.2.1':
resolution: {integrity: sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==}
@@ -10391,6 +10418,9 @@ packages:
resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==}
engines: {node: '>=10.0.0'}
'@xstate/fsm@1.6.5':
resolution: {integrity: sha512-b5o1I6aLNeYlU/3CPlj/Z91ybk1gUsKT+5NAJI+2W4UjvS5KLG28K9v5UvNoFVjHV8PajVZ00RH3vnjyQO7ZAw==}
abbrev@2.0.0:
resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -10782,6 +10812,10 @@ packages:
base-64@1.0.0:
resolution: {integrity: sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==}
base64-arraybuffer@1.0.2:
resolution: {integrity: sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==}
engines: {node: '>= 0.6.0'}
base64-js@0.0.2:
resolution: {integrity: sha512-Pj9L87dCdGcKlSqPVUjD+q96pbIx1zQQLb2CUiWURfjiBELv84YX+0nGnKmyT/9KkC7PQk7UN1w+Al8bBozaxQ==}
engines: {node: '>= 0.4'}
@@ -12679,6 +12713,9 @@ packages:
fetchdts@0.1.7:
resolution: {integrity: sha512-YoZjBdafyLIop9lSxXVI33oLD5kN31q4Td+CasofLLYeLXRFeOsuOw0Uo+XNRi9PZlbfdlN2GmRtm4tCEQ9/KA==}
fflate@0.4.8:
resolution: {integrity: sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==}
fifo@2.4.1:
resolution: {integrity: sha512-XTbUCNmo54Jav0hcL6VxDuY4x1eCQH61HEF80C2Oww283pfjQ2C8avZeyq4v43sW2S2403kmzssE9j4lbF66Sg==}
@@ -16943,9 +16980,21 @@ packages:
resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==}
engines: {node: '>= 18'}
rrdom@2.0.0-alpha.20:
resolution: {integrity: sha512-hoqjS4662LtBp82qEz9GrqU36UpEmCvTA2Hns3qdF7cklLFFy3G+0Th8hLytJENleHHWxsB5nWJ3eXz5mSRxdQ==}
rrweb-cssom@0.8.0:
resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==}
rrweb-player@2.0.0-alpha.20:
resolution: {integrity: sha512-3ZCv1ksUxuIOn3Vn/eWrwWs9Xy+4KVjISD+q26ZLfisZ3hZ0CPgYG3iC22pmmycIeMS2svOfvf7gPh7jExwpUA==}
rrweb-snapshot@2.0.0-alpha.20:
resolution: {integrity: sha512-YTNf9YVeaGRo/jxY3FKBge2c/Ojd/KTHmuWloUSB+oyPXuY73ZeeG873qMMmhIpqEn7hn7aBF1eWEQmP7wjf8A==}
rrweb@2.0.0-alpha.20:
resolution: {integrity: sha512-CZKDlm+j1VA50Ko3gnMbpvguCAleljsTNXPnVk9aeNP8o6T6kolRbISHyDZpqZ4G+bdDLlQOignPP3jEsXs8Gg==}
run-applescript@7.1.0:
resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==}
engines: {node: '>=18'}
@@ -28030,6 +28079,20 @@ snapshots:
'@rollup/rollup-win32-x64-msvc@4.52.5':
optional: true
'@rrweb/packer@2.0.0-alpha.20':
dependencies:
'@rrweb/types': 2.0.0-alpha.20
fflate: 0.4.8
'@rrweb/replay@2.0.0-alpha.20':
dependencies:
'@rrweb/types': 2.0.0-alpha.20
rrweb: 2.0.0-alpha.20
'@rrweb/types@2.0.0-alpha.20': {}
'@rrweb/utils@2.0.0-alpha.20': {}
'@segment/loosely-validate-event@2.0.0':
dependencies:
component-type: 1.2.2
@@ -29384,6 +29447,8 @@ snapshots:
'@tsconfig/node18@1.0.3': {}
'@tsconfig/svelte@1.0.13': {}
'@turf/boolean-point-in-polygon@6.5.0':
dependencies:
'@turf/helpers': 6.5.0
@@ -29474,6 +29539,8 @@ snapshots:
dependencies:
'@types/node': 20.19.24
'@types/css-font-loading-module@0.0.7': {}
'@types/d3-array@3.2.1': {}
'@types/d3-axis@3.0.6':
@@ -30364,6 +30431,8 @@ snapshots:
'@xmldom/xmldom@0.8.10': {}
'@xstate/fsm@1.6.5': {}
abbrev@2.0.0: {}
abbrev@3.0.1: {}
@@ -30909,6 +30978,8 @@ snapshots:
base-64@1.0.0: {}
base64-arraybuffer@1.0.2: {}
base64-js@0.0.2: {}
base64-js@1.5.1: {}
@@ -33466,6 +33537,8 @@ snapshots:
fetchdts@0.1.7: {}
fflate@0.4.8: {}
fifo@2.4.1: {}
figures@5.0.0:
@@ -39012,8 +39085,33 @@ snapshots:
transitivePeerDependencies:
- supports-color
rrdom@2.0.0-alpha.20:
dependencies:
rrweb-snapshot: 2.0.0-alpha.20
rrweb-cssom@0.8.0: {}
rrweb-player@2.0.0-alpha.20:
dependencies:
'@rrweb/packer': 2.0.0-alpha.20
'@rrweb/replay': 2.0.0-alpha.20
'@tsconfig/svelte': 1.0.13
rrweb-snapshot@2.0.0-alpha.20:
dependencies:
postcss: 8.5.6
rrweb@2.0.0-alpha.20:
dependencies:
'@rrweb/types': 2.0.0-alpha.20
'@rrweb/utils': 2.0.0-alpha.20
'@types/css-font-loading-module': 0.0.7
'@xstate/fsm': 1.6.5
base64-arraybuffer: 1.0.2
mitt: 3.0.1
rrdom: 2.0.0-alpha.20
rrweb-snapshot: 2.0.0-alpha.20
run-applescript@7.1.0: {}
run-async@2.4.1: {}

41
test.ts
View File

@@ -1,41 +0,0 @@
const text =
'Now I want you to create a new comparison, we should compare OpenPanel to %s. Do a deep research of %s and then create our structured json output with your result.';
const competitors = [
// Top-tier mainstream analytics (very high popularity / broad usage)
'Google Analytics', // GA4 is still the most widely used web analytics tool worldwide :contentReference[oaicite:1]{index=1}
'Mixpanel', // Widely used for product/event analytics, large customer base and market share :contentReference[oaicite:2]{index=2}
'Amplitude', // Frequently shows up among top product analytics tools in 2025 rankings :contentReference[oaicite:3]{index=3}
// Well-established alternatives (recognized, used by many, good balance of features/privacy/hosting)
'Matomo', // Open-source, powers 1M+ websites globally — leading ethical/self-hosted alternative :contentReference[oaicite:4]{index=4}
'PostHog', // Rising in popularity as a GA4 alternative with both web & product analytics, event-based tracking, self-hostable :contentReference[oaicite:5]{index=5}
'Heap', // Known in analytics rankings among top tools, often offers flexible event & session analytics :contentReference[oaicite:6]{index=6}
// Privacy-first / open-source or self-hosted lightweight solutions (gaining traction, niche but relevant)
'Plausible', // Frequently recommended as lightweight, GDPR-friendly, privacy-aware analytics alternative :contentReference[oaicite:7]{index=7}
'Fathom Analytics', // Another privacy-centric alternative often listed among top GA-alternatives :contentReference[oaicite:8]{index=8}
'Umami', // Lightweight open-source analytics; listed among top self-hosted / privacy-aware tools in 2025 reviews :contentReference[oaicite:9]{index=9}
'Kissmetrics', // Long-time product/behaviour analytics tool, still appears in “top analytics tools” listings :contentReference[oaicite:10]{index=10}
'Hotjar', // Popular for heatmaps / session recordings / user behavior insights — often used alongside analytics for qualitative data :contentReference[oaicite:11]{index=11}
// More niche, specialized or less widely adopted (but still valid alternatives / complements)
'Simple Analytics',
'GoatCounter',
'Pirsch Analytics',
'Cabin Analytics',
'Ackee',
'FullStory',
'LogRocket',
'Adobe Analytics', // Enterprise-grade, deep integration — strong reputation but more expensive and targeted at larger orgs :contentReference[oaicite:12]{index=12},
'Countly',
'Appsflyer',
'Adjust',
'Smartlook',
'Mouseflow',
'Crazy Egg',
'Microsoft Clarity',
];
for (const competitor of competitors) {
console.log('--------------------------------');
console.log(text.replaceAll('%s', competitor));
}