import { cacheable } from '@openpanel/redis'; import type { IChartEventFilter } from '@openpanel/validation'; import sqlstring from 'sqlstring'; import { TABLE_NAMES, ch, chQuery, convertClickhouseDateToJs, formatClickhouseDate, } from '../clickhouse/client'; import { clix } from '../clickhouse/query-builder'; import { createSqlBuilder } from '../sql-builder'; import { getEventFiltersWhereClause } from './chart.service'; import { getOrganizationByProjectIdCached } from './organization.service'; import { type IServiceProfile, getProfilesCached } from './profile.service'; export type IClickhouseSession = { id: string; profile_id: string; event_count: number; screen_view_count: number; screen_views: string[]; entry_path: string; entry_origin: string; exit_path: string; exit_origin: string; created_at: string; ended_at: string; referrer: string; referrer_name: string; referrer_type: string; os: string; os_version: string; browser: string; browser_version: string; device: string; brand: string; model: string; country: string; region: string; city: string; longitude: number | null; latitude: number | null; is_bounce: boolean; project_id: string; device_id: string; duration: number; utm_medium: string; utm_source: string; utm_campaign: string; utm_content: string; utm_term: string; revenue: number; sign: 1 | 0; version: number; has_replay: boolean; }; export interface IServiceSession { id: string; profileId: string; eventCount: number; screenViewCount: number; entryPath: string; entryOrigin: string; exitPath: string; exitOrigin: string; createdAt: Date; endedAt: Date; referrer: string; referrerName: string; referrerType: string; os: string; osVersion: string; browser: string; browserVersion: string; device: string; brand: string; model: string; country: string; region: string; city: string; longitude: number | null; latitude: number | null; isBounce: boolean; projectId: string; deviceId: string; duration: number; utmMedium: string; utmSource: string; utmCampaign: string; utmContent: string; utmTerm: string; revenue: number; hasReplay: boolean; profile?: IServiceProfile; } export interface GetSessionListOptions { projectId: string; profileId?: string; take: number; filters?: IChartEventFilter[]; startDate?: Date; endDate?: Date; search?: string; cursor?: Cursor | null; } export function transformSession(session: IClickhouseSession): IServiceSession { return { id: session.id, profileId: session.profile_id, eventCount: session.event_count, screenViewCount: session.screen_view_count, entryPath: session.entry_path, entryOrigin: session.entry_origin, exitPath: session.exit_path, exitOrigin: session.exit_origin, createdAt: convertClickhouseDateToJs(session.created_at), endedAt: convertClickhouseDateToJs(session.ended_at), referrer: session.referrer, referrerName: session.referrer_name, referrerType: session.referrer_type, os: session.os, osVersion: session.os_version, browser: session.browser, browserVersion: session.browser_version, device: session.device, brand: session.brand, model: session.model, country: session.country, region: session.region, city: session.city, longitude: session.longitude, latitude: session.latitude, isBounce: session.is_bounce, projectId: session.project_id, deviceId: session.device_id, duration: session.duration, utmMedium: session.utm_medium, utmSource: session.utm_source, utmCampaign: session.utm_campaign, utmContent: session.utm_content, utmTerm: session.utm_term, revenue: session.revenue, hasReplay: session.has_replay, profile: undefined, }; } type Direction = 'initial' | 'next' | 'prev'; type PageInfo = { next?: Cursor; // use last row }; type Cursor = { createdAt: string; // ISO 8601 with ms id: string; }; export async function getSessionList({ cursor, take, projectId, profileId, filters, startDate, endDate, search, }: GetSessionListOptions) { const { sb, getSql } = createSqlBuilder(); sb.from = `${TABLE_NAMES.sessions} FINAL`; sb.limit = take; sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`; if (startDate && endDate) { sb.where.range = `created_at BETWEEN toDateTime('${formatClickhouseDate(startDate)}') AND toDateTime('${formatClickhouseDate(endDate)}')`; } if (profileId) sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`; if (search) { const s = sqlstring.escape(`%${search}%`); sb.where.search = `(entry_path ILIKE ${s} OR exit_path ILIKE ${s} OR referrer ILIKE ${s} OR referrer_name ILIKE ${s})`; } if (filters?.length) { Object.assign(sb.where, getEventFiltersWhereClause(filters)); } const organization = await getOrganizationByProjectIdCached(projectId); // This will speed up the query quite a lot for big organizations const dateIntervalInDays = organization?.subscriptionPeriodEventsLimit && organization?.subscriptionPeriodEventsLimit > 1_000_000 ? 2 : 360; if (cursor) { const cAt = sqlstring.escape(cursor.createdAt); sb.where.cursor = `created_at < toDateTime64(${cAt}, 3)`; sb.where.cursorWindow = `created_at >= toDateTime64(${cAt}, 3) - INTERVAL ${dateIntervalInDays} DAY`; sb.orderBy.created_at = 'created_at DESC'; } else { sb.orderBy.created_at = 'created_at DESC'; sb.where.created_at = `created_at > now() - INTERVAL ${dateIntervalInDays} DAY`; } // ==== Select columns (as you had) ==== // sb.select.id = 'id'; sb.select.project_id = 'project_id'; ... etc. const columns = [ 'created_at', 'ended_at', 'id', 'profile_id', 'entry_path', 'exit_path', 'duration', 'is_bounce', 'referrer_name', 'referrer', 'country', 'city', 'os', 'browser', 'brand', 'model', 'device', 'screen_view_count', 'event_count', 'revenue', ]; columns.forEach((column) => { sb.select[column] = column; }); sb.select.has_replay = `exists(SELECT 1 FROM ${TABLE_NAMES.session_replay_chunks} WHERE session_id = id AND project_id = ${sqlstring.escape(projectId)}) as has_replay`; const sql = getSql(); const data = await chQuery< IClickhouseSession & { latestCreatedAt: string; } >(sql); // Compute cursors from page edges const last = data[take - 1]; const meta: PageInfo = { next: last ? { createdAt: last.created_at, id: last.id, } : undefined, }; // Profile hydration (unchanged) const profileIds = data .filter((e) => e.device_id !== e.profile_id) .map((e) => e.profile_id); const profiles = await getProfilesCached(profileIds, projectId); const map = new Map(profiles.map((p) => [p.id, p])); const items = data.map(transformSession).map((item) => ({ ...item, profile: map.get(item.profileId) ?? { id: item.profileId, email: '', avatar: '', firstName: '', lastName: '', createdAt: new Date(), projectId, isExternal: false, properties: {}, }, })); return { items, meta }; } export async function getSessionsCount({ projectId, profileId, filters, startDate, endDate, search, }: Omit) { const { sb, getSql } = createSqlBuilder(); sb.select.count = 'count(*) as count'; sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`; sb.where.sign = 'sign = 1'; if (profileId) { sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`; } if (startDate && endDate) { sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`; } if (search) { sb.where.search = `(entry_path ILIKE '%${search}%' OR exit_path ILIKE '%${search}%' OR referrer ILIKE '%${search}%' OR referrer_name ILIKE '%${search}%')`; } if (filters && filters.length > 0) { const sessionFilters = getEventFiltersWhereClause(filters); sb.where = { ...sb.where, ...sessionFilters, }; } sb.from = TABLE_NAMES.sessions; const result = await chQuery<{ count: number }>(getSql()); return result[0]?.count ?? 0; } export const getSessionsCountCached = cacheable(getSessionsCount, 60 * 10); export async function getSessionReplayEvents( sessionId: string, projectId: string, ): Promise<{ events: unknown[] }> { const chunks = await clix(ch) .select<{ chunk_index: number; payload: string }>([ 'chunk_index', 'payload', ]) .from(TABLE_NAMES.session_replay_chunks) .where('session_id', '=', sessionId) .where('project_id', '=', projectId) .orderBy('chunk_index', 'ASC') .execute(); const allEvents = chunks.flatMap( (chunk) => JSON.parse(chunk.payload) as unknown[], ); // rrweb event types: 2 = FullSnapshot, 4 = Meta // Incremental snapshots (type 3) before the first FullSnapshot are orphaned // and cause the player to fast-forward through empty time. Strip them but // keep Meta events (type 4) since rrweb needs them for viewport dimensions. const firstFullSnapshotIdx = allEvents.findIndex((e: any) => e.type === 2); let events = allEvents; if (firstFullSnapshotIdx > 0) { const metaEvents = allEvents .slice(0, firstFullSnapshotIdx) .filter((e: any) => e.type === 4); events = [...metaEvents, ...allEvents.slice(firstFullSnapshotIdx)]; } return { events }; } class SessionService { constructor(private client: typeof ch) {} async byId(sessionId: string, projectId: string) { const result = await clix(this.client) .select(['*']) .from(TABLE_NAMES.sessions) .where('id', '=', sessionId) .where('project_id', '=', projectId) .where('sign', '=', 1) .execute(); if (!result[0]) { throw new Error('Session not found'); } return transformSession(result[0]); } } export const sessionService = new SessionService(ch);