diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts index 1629c591..4c03c67b 100644 --- a/apps/api/src/controllers/track.controller.ts +++ b/apps/api/src/controllers/track.controller.ts @@ -3,11 +3,7 @@ import { assocPath, pathOr, pick } from 'ramda'; import { HttpError } from '@/utils/errors'; import { generateId, slug } from '@openpanel/common'; -import { - generateDeviceId, - generateSecureId, - parseUserAgent, -} from '@openpanel/common/server'; +import { generateDeviceId, parseUserAgent } from '@openpanel/common/server'; import { TABLE_NAMES, ch, @@ -20,6 +16,7 @@ import { type GeoLocation, getGeoLocation } from '@openpanel/geo'; import { getEventsGroupQueueShard } from '@openpanel/queue'; import { getRedisCache } from '@openpanel/redis'; +import { getDeviceId } from '@/utils/ids'; import { type IDecrementPayload, type IIdentifyPayload, @@ -30,84 +27,6 @@ import { zTrackHandlerPayload, } from '@openpanel/validation'; -async function getDeviceId({ - projectId, - ip, - ua, - salts, - overrideDeviceId, -}: { - projectId: string; - ip: string; - ua: string | undefined; - salts: { current: string; previous: string }; - overrideDeviceId?: string; -}) { - if (overrideDeviceId) { - return { deviceId: overrideDeviceId, sessionId: undefined }; - } - - if (!ua) { - return { deviceId: '', sessionId: undefined }; - } - - const currentDeviceId = generateDeviceId({ - salt: salts.current, - origin: projectId, - ip, - ua, - }); - const previousDeviceId = generateDeviceId({ - salt: salts.previous, - origin: projectId, - ip, - ua, - }); - - return await getDeviceIdFromSession({ - projectId, - currentDeviceId, - previousDeviceId, - }); -} - -async function getDeviceIdFromSession({ - projectId, - currentDeviceId, - previousDeviceId, -}: { - projectId: string; - currentDeviceId: string; - previousDeviceId: string; -}) { - try { - const multi = getRedisCache().multi(); - multi.hget( - `bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`, - 'data', - ); - multi.hget( - `bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`, - 'data', - ); - const res = await multi.exec(); - if (res?.[0]?.[1]) { - const data = JSON.parse(res?.[0]?.[1] as string); - const sessionId = data.payload.sessionId; - return { deviceId: currentDeviceId, sessionId }; - } - if (res?.[1]?.[1]) { - const data = JSON.parse(res?.[1]?.[1] as string); - const sessionId = data.payload.sessionId; - return { deviceId: previousDeviceId, sessionId }; - } - } catch (error) { - console.error('Error getting session end GET /track/device-id', error); - } - - return { deviceId: currentDeviceId, sessionId: generateSecureId('se') }; -} - export function getStringHeaders(headers: FastifyRequest['headers']) { return Object.entries( pick( diff --git a/apps/api/src/utils/ids.ts b/apps/api/src/utils/ids.ts new file mode 100644 index 00000000..3944df6e --- /dev/null +++ b/apps/api/src/utils/ids.ts @@ -0,0 +1,141 @@ +import crypto from 'node:crypto'; +import { generateDeviceId } from '@openpanel/common/server'; +import { getRedisCache } from '@openpanel/redis'; + +export async function getDeviceId({ + projectId, + ip, + ua, + salts, + overrideDeviceId, +}: { + projectId: string; + ip: string; + ua: string | undefined; + salts: { current: string; previous: string }; + overrideDeviceId?: string; +}) { + if (overrideDeviceId) { + return { deviceId: overrideDeviceId, sessionId: undefined }; + } + + if (!ua) { + return { deviceId: '', sessionId: undefined }; + } + + const currentDeviceId = generateDeviceId({ + salt: salts.current, + origin: projectId, + ip, + ua, + }); + const previousDeviceId = generateDeviceId({ + salt: salts.previous, + origin: projectId, + ip, + ua, + }); + + return await getDeviceIdFromSession({ + projectId, + currentDeviceId, + previousDeviceId, + }); +} + +async function getDeviceIdFromSession({ + projectId, + currentDeviceId, + previousDeviceId, +}: { + projectId: string; + currentDeviceId: string; + previousDeviceId: string; +}) { + try { + const multi = getRedisCache().multi(); + multi.hget( + `bull:sessions:sessionEnd:${projectId}:${currentDeviceId}`, + 'data', + ); + multi.hget( + `bull:sessions:sessionEnd:${projectId}:${previousDeviceId}`, + 'data', + ); + const res = await multi.exec(); + if (res?.[0]?.[1]) { + const data = JSON.parse(res?.[0]?.[1] as string); + const sessionId = data.payload.sessionId; + return { deviceId: currentDeviceId, sessionId }; + } + if (res?.[1]?.[1]) { + const data = JSON.parse(res?.[1]?.[1] as string); + const sessionId = data.payload.sessionId; + return { deviceId: previousDeviceId, sessionId }; + } + } catch (error) { + console.error('Error getting session end GET /track/device-id', error); + } + + return { + deviceId: currentDeviceId, + sessionId: getSessionId({ + projectId, + deviceId: currentDeviceId, + graceMs: 5 * 1000, + windowMs: 1000 * 60 * 30, + }), + }; +} + +/** + * Deterministic session id for (projectId, deviceId) within a time window, + * with a grace period at the *start* of each window to avoid boundary splits. + * + * - windowMs: 5 minutes by default + * - graceMs: 1 minute by default (events in first minute of a bucket map to previous bucket) + * - Output: base64url, 128-bit (16 bytes) truncated from SHA-256 + */ +function getSessionId(params: { + projectId: string; + deviceId: string; + eventMs?: number; // use event timestamp; defaults to Date.now() + windowMs?: number; // default 5 min + graceMs?: number; // default 1 min + bytes?: number; // default 16 (128-bit). You can set 24 or 32 for longer ids. +}): string { + const { + projectId, + deviceId, + eventMs = Date.now(), + windowMs = 5 * 60 * 1000, + graceMs = 60 * 1000, + bytes = 16, + } = params; + + if (!projectId) throw new Error('projectId is required'); + if (!deviceId) throw new Error('deviceId is required'); + if (windowMs <= 0) throw new Error('windowMs must be > 0'); + if (graceMs < 0 || graceMs >= windowMs) + throw new Error('graceMs must be >= 0 and < windowMs'); + if (bytes < 8 || bytes > 32) + throw new Error('bytes must be between 8 and 32'); + + const bucket = Math.floor(eventMs / windowMs); + const offset = eventMs - bucket * windowMs; + + // Grace at the start of the bucket: stick to the previous bucket. + const chosenBucket = offset < graceMs ? bucket - 1 : bucket; + + const input = `sess:v1:${projectId}:${deviceId}:${chosenBucket}`; + + const digest = crypto.createHash('sha256').update(input).digest(); + const truncated = digest.subarray(0, bytes); + + // base64url + return truncated + .toString('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/g, ''); +} diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.ts index 0e2e9ce9..ec98aa95 100644 --- a/packages/db/src/services/session.service.ts +++ b/packages/db/src/services/session.service.ts @@ -194,13 +194,11 @@ export async function getSessionList({ const dateIntervalInDays = organization?.subscriptionPeriodEventsLimit && organization?.subscriptionPeriodEventsLimit > 1_000_000 - ? 1 + ? 2 : 360; if (cursor) { const cAt = sqlstring.escape(cursor.createdAt); - // TODO: remove id from cursor - const cId = sqlstring.escape(cursor.id); sb.where.cursor = `created_at < toDateTime64(${cAt}, 3)`; sb.where.cursorWindow = `created_at >= toDateTime64(${cAt}, 3) - INTERVAL ${dateIntervalInDays} DAY`; sb.orderBy.created_at = 'created_at DESC'; @@ -330,24 +328,25 @@ export async function getSessionReplayEvents( projectId: string, ): Promise<{ events: unknown[] }> { const chunks = await clix(ch) - .select<{ chunk_index: number; payload: string }>(['chunk_index', 'payload']) + .select<{ chunk_index: number; payload: string }>([ + 'chunk_index', + 'payload', + ]) .from(TABLE_NAMES.session_replay_chunks) .where('session_id', '=', sessionId) .where('project_id', '=', projectId) .orderBy('chunk_index', 'ASC') .execute(); - const allEvents = chunks.flatMap((chunk) => - JSON.parse(chunk.payload) as unknown[], + const allEvents = chunks.flatMap( + (chunk) => JSON.parse(chunk.payload) as unknown[], ); // rrweb event types: 2 = FullSnapshot, 4 = Meta // Incremental snapshots (type 3) before the first FullSnapshot are orphaned // and cause the player to fast-forward through empty time. Strip them but // keep Meta events (type 4) since rrweb needs them for viewport dimensions. - const firstFullSnapshotIdx = allEvents.findIndex( - (e: any) => e.type === 2, - ); + const firstFullSnapshotIdx = allEvents.findIndex((e: any) => e.type === 2); let events = allEvents; if (firstFullSnapshotIdx > 0) { diff --git a/packages/sdks/web/package.json b/packages/sdks/web/package.json index 8de98203..7f5fb1e2 100644 --- a/packages/sdks/web/package.json +++ b/packages/sdks/web/package.json @@ -10,7 +10,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@openpanel/sdk": "workspace:1.0.4-local" + "@openpanel/sdk": "workspace:1.0.4-local", "@rrweb/types": "2.0.0-alpha.20", "rrweb": "2.0.0-alpha.20" }, @@ -20,4 +20,4 @@ "tsup": "^7.2.0", "typescript": "catalog:" } -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c50f9b1a..ab319d2d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -774,6 +774,9 @@ importers: remark-rehype: specifier: ^11.1.2 version: 11.1.2 + rrweb-player: + specifier: 2.0.0-alpha.20 + version: 2.0.0-alpha.20 short-unique-id: specifier: ^5.0.3 version: 5.0.3 @@ -1629,6 +1632,12 @@ importers: '@openpanel/sdk': specifier: workspace:1.0.4-local version: link:../sdk + '@rrweb/types': + specifier: 2.0.0-alpha.20 + version: 2.0.0-alpha.20 + rrweb: + specifier: 2.0.0-alpha.20 + version: 2.0.0-alpha.20 devDependencies: '@openpanel/tsconfig': specifier: workspace:* @@ -8705,6 +8714,18 @@ packages: cpu: [x64] os: [win32] + '@rrweb/packer@2.0.0-alpha.20': + resolution: {integrity: sha512-GsByg2olGZ2n3To6keFG604QAboipuXZvjYxO2wITSwARBf/sZdy6cbUEjF0RS+QnuTM5GaVXeQapNMLmpKbrA==} + + '@rrweb/replay@2.0.0-alpha.20': + resolution: {integrity: sha512-VodsLb+C2bYNNVbb0U14tKLa9ctzUxYIlt9VnxPATWvfyXHLTku8BhRWptuW/iIjVjmG49LBoR1ilxw/HMiJ1w==} + + '@rrweb/types@2.0.0-alpha.20': + resolution: {integrity: sha512-RbnDgKxA/odwB1R4gF7eUUj+rdSrq6ROQJsnMw7MIsGzlbSYvJeZN8YY4XqU0G6sKJvXI6bSzk7w/G94jNwzhw==} + + '@rrweb/utils@2.0.0-alpha.20': + resolution: {integrity: sha512-MTQOmhPRe39C0fYaCnnVYOufQsyGzwNXpUStKiyFSfGLUJrzuwhbRoUAKR5w6W2j5XuA0bIz3ZDIBztkquOhLw==} + '@segment/loosely-validate-event@2.0.0': resolution: {integrity: sha512-ZMCSfztDBqwotkl848ODgVcAmN4OItEWDCkshcKz0/W6gGSQayuuCtWV/MlodFivAZD793d6UgANd6wCXUfrIw==} @@ -9656,6 +9677,9 @@ packages: '@tsconfig/node18@1.0.3': resolution: {integrity: sha512-RbwvSJQsuN9TB04AQbGULYfOGE/RnSFk/FLQ5b0NmDf5Kx2q/lABZbHQPKCO1vZ6Fiwkplu+yb9pGdLy1iGseQ==} + '@tsconfig/svelte@1.0.13': + resolution: {integrity: sha512-5lYJP45Xllo4yE/RUBccBT32eBlRDbqN8r1/MIvQbKxW3aFqaYPCNgm8D5V20X4ShHcwvYWNlKg3liDh1MlBoA==} + '@turf/boolean-point-in-polygon@6.5.0': resolution: {integrity: sha512-DtSuVFB26SI+hj0SjrvXowGTUCHlgevPAIsukssW6BG5MlNSBQAo70wpICBNJL6RjukXg8d2eXaAWuD/CqL00A==} @@ -9722,6 +9746,9 @@ packages: '@types/cors@2.8.17': resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==} + '@types/css-font-loading-module@0.0.7': + resolution: {integrity: sha512-nl09VhutdjINdWyXxHWN/w9zlNCfr60JUqJbd24YXUuCwgeL0TpFSdElCwb6cxfB6ybE19Gjj4g0jsgkXxKv1Q==} + '@types/d3-array@3.2.1': resolution: {integrity: sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==} @@ -10391,6 +10418,9 @@ packages: resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==} engines: {node: '>=10.0.0'} + '@xstate/fsm@1.6.5': + resolution: {integrity: sha512-b5o1I6aLNeYlU/3CPlj/Z91ybk1gUsKT+5NAJI+2W4UjvS5KLG28K9v5UvNoFVjHV8PajVZ00RH3vnjyQO7ZAw==} + abbrev@2.0.0: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -10782,6 +10812,10 @@ packages: base-64@1.0.0: resolution: {integrity: sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==} + base64-arraybuffer@1.0.2: + resolution: {integrity: sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==} + engines: {node: '>= 0.6.0'} + base64-js@0.0.2: resolution: {integrity: sha512-Pj9L87dCdGcKlSqPVUjD+q96pbIx1zQQLb2CUiWURfjiBELv84YX+0nGnKmyT/9KkC7PQk7UN1w+Al8bBozaxQ==} engines: {node: '>= 0.4'} @@ -12679,6 +12713,9 @@ packages: fetchdts@0.1.7: resolution: {integrity: sha512-YoZjBdafyLIop9lSxXVI33oLD5kN31q4Td+CasofLLYeLXRFeOsuOw0Uo+XNRi9PZlbfdlN2GmRtm4tCEQ9/KA==} + fflate@0.4.8: + resolution: {integrity: sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==} + fifo@2.4.1: resolution: {integrity: sha512-XTbUCNmo54Jav0hcL6VxDuY4x1eCQH61HEF80C2Oww283pfjQ2C8avZeyq4v43sW2S2403kmzssE9j4lbF66Sg==} @@ -16943,9 +16980,21 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + rrdom@2.0.0-alpha.20: + resolution: {integrity: sha512-hoqjS4662LtBp82qEz9GrqU36UpEmCvTA2Hns3qdF7cklLFFy3G+0Th8hLytJENleHHWxsB5nWJ3eXz5mSRxdQ==} + rrweb-cssom@0.8.0: resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==} + rrweb-player@2.0.0-alpha.20: + resolution: {integrity: sha512-3ZCv1ksUxuIOn3Vn/eWrwWs9Xy+4KVjISD+q26ZLfisZ3hZ0CPgYG3iC22pmmycIeMS2svOfvf7gPh7jExwpUA==} + + rrweb-snapshot@2.0.0-alpha.20: + resolution: {integrity: sha512-YTNf9YVeaGRo/jxY3FKBge2c/Ojd/KTHmuWloUSB+oyPXuY73ZeeG873qMMmhIpqEn7hn7aBF1eWEQmP7wjf8A==} + + rrweb@2.0.0-alpha.20: + resolution: {integrity: sha512-CZKDlm+j1VA50Ko3gnMbpvguCAleljsTNXPnVk9aeNP8o6T6kolRbISHyDZpqZ4G+bdDLlQOignPP3jEsXs8Gg==} + run-applescript@7.1.0: resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} engines: {node: '>=18'} @@ -28030,6 +28079,20 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.52.5': optional: true + '@rrweb/packer@2.0.0-alpha.20': + dependencies: + '@rrweb/types': 2.0.0-alpha.20 + fflate: 0.4.8 + + '@rrweb/replay@2.0.0-alpha.20': + dependencies: + '@rrweb/types': 2.0.0-alpha.20 + rrweb: 2.0.0-alpha.20 + + '@rrweb/types@2.0.0-alpha.20': {} + + '@rrweb/utils@2.0.0-alpha.20': {} + '@segment/loosely-validate-event@2.0.0': dependencies: component-type: 1.2.2 @@ -29384,6 +29447,8 @@ snapshots: '@tsconfig/node18@1.0.3': {} + '@tsconfig/svelte@1.0.13': {} + '@turf/boolean-point-in-polygon@6.5.0': dependencies: '@turf/helpers': 6.5.0 @@ -29474,6 +29539,8 @@ snapshots: dependencies: '@types/node': 20.19.24 + '@types/css-font-loading-module@0.0.7': {} + '@types/d3-array@3.2.1': {} '@types/d3-axis@3.0.6': @@ -30364,6 +30431,8 @@ snapshots: '@xmldom/xmldom@0.8.10': {} + '@xstate/fsm@1.6.5': {} + abbrev@2.0.0: {} abbrev@3.0.1: {} @@ -30909,6 +30978,8 @@ snapshots: base-64@1.0.0: {} + base64-arraybuffer@1.0.2: {} + base64-js@0.0.2: {} base64-js@1.5.1: {} @@ -33466,6 +33537,8 @@ snapshots: fetchdts@0.1.7: {} + fflate@0.4.8: {} + fifo@2.4.1: {} figures@5.0.0: @@ -39012,8 +39085,33 @@ snapshots: transitivePeerDependencies: - supports-color + rrdom@2.0.0-alpha.20: + dependencies: + rrweb-snapshot: 2.0.0-alpha.20 + rrweb-cssom@0.8.0: {} + rrweb-player@2.0.0-alpha.20: + dependencies: + '@rrweb/packer': 2.0.0-alpha.20 + '@rrweb/replay': 2.0.0-alpha.20 + '@tsconfig/svelte': 1.0.13 + + rrweb-snapshot@2.0.0-alpha.20: + dependencies: + postcss: 8.5.6 + + rrweb@2.0.0-alpha.20: + dependencies: + '@rrweb/types': 2.0.0-alpha.20 + '@rrweb/utils': 2.0.0-alpha.20 + '@types/css-font-loading-module': 0.0.7 + '@xstate/fsm': 1.6.5 + base64-arraybuffer: 1.0.2 + mitt: 3.0.1 + rrdom: 2.0.0-alpha.20 + rrweb-snapshot: 2.0.0-alpha.20 + run-applescript@7.1.0: {} run-async@2.4.1: {} diff --git a/test.ts b/test.ts deleted file mode 100644 index 81947e9e..00000000 --- a/test.ts +++ /dev/null @@ -1,41 +0,0 @@ -const text = - 'Now I want you to create a new comparison, we should compare OpenPanel to %s. Do a deep research of %s and then create our structured json output with your result.'; - -const competitors = [ - // Top-tier mainstream analytics (very high popularity / broad usage) - 'Google Analytics', // GA4 is still the most widely used web analytics tool worldwide :contentReference[oaicite:1]{index=1} - 'Mixpanel', // Widely used for product/event analytics, large customer base and market share :contentReference[oaicite:2]{index=2} - 'Amplitude', // Frequently shows up among top product analytics tools in 2025 rankings :contentReference[oaicite:3]{index=3} - // Well-established alternatives (recognized, used by many, good balance of features/privacy/hosting) - 'Matomo', // Open-source, powers 1M+ websites globally — leading ethical/self-hosted alternative :contentReference[oaicite:4]{index=4} - 'PostHog', // Rising in popularity as a GA4 alternative with both web & product analytics, event-based tracking, self-hostable :contentReference[oaicite:5]{index=5} - 'Heap', // Known in analytics rankings among top tools, often offers flexible event & session analytics :contentReference[oaicite:6]{index=6} - - // Privacy-first / open-source or self-hosted lightweight solutions (gaining traction, niche but relevant) - 'Plausible', // Frequently recommended as lightweight, GDPR-friendly, privacy-aware analytics alternative :contentReference[oaicite:7]{index=7} - 'Fathom Analytics', // Another privacy-centric alternative often listed among top GA-alternatives :contentReference[oaicite:8]{index=8} - 'Umami', // Lightweight open-source analytics; listed among top self-hosted / privacy-aware tools in 2025 reviews :contentReference[oaicite:9]{index=9} - 'Kissmetrics', // Long-time product/behaviour analytics tool, still appears in “top analytics tools” listings :contentReference[oaicite:10]{index=10} - 'Hotjar', // Popular for heatmaps / session recordings / user behavior insights — often used alongside analytics for qualitative data :contentReference[oaicite:11]{index=11} - // More niche, specialized or less widely adopted (but still valid alternatives / complements) - 'Simple Analytics', - 'GoatCounter', - 'Pirsch Analytics', - 'Cabin Analytics', - 'Ackee', - 'FullStory', - 'LogRocket', - 'Adobe Analytics', // Enterprise-grade, deep integration — strong reputation but more expensive and targeted at larger orgs :contentReference[oaicite:12]{index=12}, - 'Countly', - 'Appsflyer', - 'Adjust', - 'Smartlook', - 'Mouseflow', - 'Crazy Egg', - 'Microsoft Clarity', -]; - -for (const competitor of competitors) { - console.log('--------------------------------'); - console.log(text.replaceAll('%s', competitor)); -}