import { omit, prop, uniqBy } from "ramda"; import { generateProfileId, getTime, toISOString } from "@openpanel/common"; import type { Event, IServiceCreateEventPayload } from "@openpanel/db"; import { createEvent as createClickhouseEvent, db, formatClickhouseDate, getSalts, } from "@openpanel/db"; import { parseIp } from "../src/utils/parseIp"; import { parseUserAgent } from "../src/utils/parseUserAgent"; const clean = omit([ "ip", "os", "ua", "url", "hash", "host", "path", "device", "screen", "hostname", "language", "referrer", "timezone", ]); async function main() { const events = await db.event.findMany({ where: { project_id: "4e2798cb-e255-4e9d-960d-c9ad095aabd7", name: "screen_view", createdAt: { gte: new Date("2024-01-01"), lt: new Date("2024-02-04"), }, }, orderBy: { createdAt: "asc", }, }); const grouped: Record = {}; let index = 0; for (const event of events.slice()) { // console.log(index, event.name, event.createdAt.toISOString()); index++; const properties = event.properties as Record; if (properties.ua?.includes("bot")) { // console.log('IGNORE', event.id); continue; } if (!event.profile_id) { // console.log('IGNORE', event.id); continue; } const hej = grouped[event.profile_id]; if (hej) { hej.push(event); } else { grouped[event.profile_id] = [event]; } } console.log("Total users", Object.keys(grouped).length); let uidx = -1; for (const profile_id of Object.keys(grouped)) { uidx++; console.log(`User index ${uidx}`); const events = uniqBy(prop("createdAt"), grouped[profile_id] || []); if (events) { let lastSessionStart = null; let screenViews = 0; let totalDuration = 0; console.log("new user..."); let eidx = -1; for (const event of events) { eidx++; const prevEvent = events[eidx - 1]; const prevEventAt = prevEvent?.createdAt; const nextEvent = events[eidx + 1]; const properties = event.properties as Record; const projectId = event.project_id; const path = properties.path!; const ip = properties.ip!; const origin = "https://openpanel.kiddo.se"; const ua = properties.ua!; const uaInfo = parseUserAgent(ua); const salts = await getSalts(); const profileId = generateProfileId({ salt: salts.current, origin, ip, ua, }); const geo = parseIp(ip); const isNextEventNewSession = nextEvent && nextEvent.createdAt.getTime() - event.createdAt.getTime() > 1000 * 60 * 30; const payload: IServiceCreateEventPayload = { name: event.name, profileId, projectId, properties: clean(properties), createdAt: event.createdAt.toISOString(), country: geo.country, city: geo.city, region: geo.region, continent: geo.continent, os: uaInfo.os, osVersion: uaInfo.osVersion, browser: uaInfo.browser, browserVersion: uaInfo.browserVersion, device: uaInfo.device, brand: uaInfo.brand, model: uaInfo.model, duration: nextEvent && !isNextEventNewSession ? nextEvent.createdAt.getTime() - event.createdAt.getTime() : 0, path, referrer: properties?.referrer?.host ?? "", // TODO referrerName: properties?.referrer?.host ?? "", // TODO }; if (!prevEventAt) { lastSessionStart = await createSessionStart(payload); } else if ( event.createdAt.getTime() - prevEventAt.getTime() > 1000 * 60 * 30 ) { if (eidx > 0 && prevEventAt && lastSessionStart) { await createSessionEnd(prevEventAt, lastSessionStart, { screenViews, totalDuration, }); totalDuration = 0; screenViews = 0; lastSessionStart = await createSessionStart(payload); } } screenViews++; totalDuration += payload.duration; await createEvent(payload); } // for each user event const prevEventAt = events[events.length - 1]?.createdAt; if (prevEventAt && lastSessionStart) { await createSessionEnd(prevEventAt, lastSessionStart, { screenViews, totalDuration, }); } } } } async function createEvent(event: IServiceCreateEventPayload) { console.log( `Create ${event.name} - ${event.path} - ${formatClickhouseDate( event.createdAt, )} - ${event.duration / 1000} sec`, ); await createClickhouseEvent(event); } async function createSessionStart(event: IServiceCreateEventPayload) { const session: IServiceCreateEventPayload = { ...event, duration: 0, name: "session_start", createdAt: toISOString(getTime(event.createdAt) - 100), }; await createEvent(session); return session; } async function createSessionEnd( prevEventAt: Date, sessionStart: IServiceCreateEventPayload, options: { screenViews: number; totalDuration: number; }, ) { const properties: Record = {}; if (options.screenViews === 1) { properties.__bounce = true; } else { properties.__bounce = false; } const session: IServiceCreateEventPayload = { ...sessionStart, properties: { ...properties, ...sessionStart.properties, }, duration: options.totalDuration, name: "session_end", createdAt: toISOString(prevEventAt.getTime() + 10), }; await createEvent(session); return session; } main();