import { endOfDay, endOfYear, startOfDay, startOfMonth, startOfYear, subDays, subMinutes, subMonths, subYears, } from 'date-fns'; import * as mathjs from 'mathjs'; import { repeat, reverse, sort } from 'ramda'; import { escape } from 'sqlstring'; import { round } from '@openpanel/common'; import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants'; import { chQuery, convertClickhouseDateToJs, createSqlBuilder, formatClickhouseDate, getChartSql, getEventFiltersWhereClause, getProfiles, } from '@openpanel/db'; import type { IChartEvent, IChartInput, IChartRange, IGetChartDataInput, IInterval, } from '@openpanel/validation'; export type GetChartDataResult = Awaited>; export interface ResultItem { label: string | null; count: number | null; date: string; } function getEventLegend(event: IChartEvent) { return event.displayName ?? `${event.name} (${event.id})`; } function fillEmptySpotsInTimeline( items: ResultItem[], interval: IInterval, startDate: string, endDate: string ) { const result = []; const clonedStartDate = new Date(startDate); const clonedEndDate = new Date(endDate); const today = new Date(); if (interval === 'minute') { clonedStartDate.setUTCSeconds(0, 0); clonedEndDate.setUTCMinutes(clonedEndDate.getUTCMinutes() + 1, 0, 0); } else if (interval === 'hour') { clonedStartDate.setUTCMinutes(0, 0, 0); clonedEndDate.setUTCMinutes(0, 0, 0); } else { clonedStartDate.setUTCHours(0, 0, 0, 0); clonedEndDate.setUTCHours(0, 0, 0, 0); } if (interval === 'month') { clonedStartDate.setUTCDate(1); clonedEndDate.setUTCDate(1); } // Force if interval is month and the start date is the same month as today const shouldForce = () => interval === 'month' && clonedStartDate.getUTCFullYear() === today.getUTCFullYear() && clonedStartDate.getUTCMonth() === today.getUTCMonth(); let prev = undefined; while ( shouldForce() || clonedStartDate.getTime() <= clonedEndDate.getTime() ) { if (prev === clonedStartDate.getTime()) { break; } prev = clonedStartDate.getTime(); const getYear = (date: Date) => date.getUTCFullYear(); const getMonth = (date: Date) => date.getUTCMonth(); const getDay = (date: Date) => date.getUTCDate(); const getHour = (date: Date) => date.getUTCHours(); const getMinute = (date: Date) => date.getUTCMinutes(); const item = items.find((item) => { const date = convertClickhouseDateToJs(item.date); if (interval === 'month') { return ( getYear(date) === getYear(clonedStartDate) && getMonth(date) === getMonth(clonedStartDate) ); } if (interval === 'day') { return ( getYear(date) === getYear(clonedStartDate) && getMonth(date) === getMonth(clonedStartDate) && getDay(date) === getDay(clonedStartDate) ); } if (interval === 'hour') { return ( getYear(date) === getYear(clonedStartDate) && getMonth(date) === getMonth(clonedStartDate) && getDay(date) === getDay(clonedStartDate) && getHour(date) === getHour(clonedStartDate) ); } if (interval === 'minute') { return ( getYear(date) === getYear(clonedStartDate) && getMonth(date) === getMonth(clonedStartDate) && getDay(date) === getDay(clonedStartDate) && getHour(date) === getHour(clonedStartDate) && getMinute(date) === getMinute(clonedStartDate) ); } return false; }); if (item) { result.push({ ...item, date: clonedStartDate.toISOString(), }); } else { result.push({ date: clonedStartDate.toISOString(), count: 0, label: null, }); } switch (interval) { case 'day': { clonedStartDate.setUTCDate(clonedStartDate.getUTCDate() + 1); break; } case 'hour': { clonedStartDate.setUTCHours(clonedStartDate.getUTCHours() + 1); break; } case 'minute': { clonedStartDate.setUTCMinutes(clonedStartDate.getUTCMinutes() + 1); break; } case 'month': { clonedStartDate.setUTCMonth(clonedStartDate.getUTCMonth() + 1); break; } } } return sort(function (a, b) { return new Date(a.date).getTime() - new Date(b.date).getTime(); }, result); } export function withFormula( { formula, events }: IChartInput, series: GetChartDataResult ) { if (!formula) { return series; } if (!series) { return series; } if (!series[0]) { return series; } if (!series[0].data) { return series; } if (events.length === 1) { return series.map((serie) => { return { ...serie, data: serie.data.map((item) => { serie.event.id; const scope = { [serie.event.id]: item?.count ?? 0, }; const count = mathjs .parse(formula) .compile() .evaluate(scope) as number; return { ...item, count: Number.isNaN(count) || !Number.isFinite(count) ? null : round(count, 2), }; }), }; }); } return [ { ...series[0], data: series[0].data.map((item, dIndex) => { const scope = series.reduce((acc, item) => { return { ...acc, [item.event.id]: item.data[dIndex]?.count ?? 0, }; }, {}); const count = mathjs.parse(formula).compile().evaluate(scope) as number; return { ...item, count: Number.isNaN(count) || !Number.isFinite(count) ? null : round(count, 2), }; }), }, ]; } export async function getChartData(payload: IGetChartDataInput) { let result = await chQuery(getChartSql(payload)); if (result.length === 0 && payload.breakdowns.length > 0) { result = await chQuery( getChartSql({ ...payload, breakdowns: [], }) ); } // group by sql label const series = result.reduce( (acc, item) => { // item.label can be null when using breakdowns on a property // that doesn't exist on all events const label = item.label?.trim() || NOT_SET_VALUE; if (label) { if (acc[label]) { acc[label]?.push(item); } else { acc[label] = [item]; } } return { ...acc, }; }, {} as Record ); return Object.keys(series).map((key) => { // If we have breakdowns, we want to use the breakdown key as the legend // But only if it successfully broke it down, otherwise we use the getEventLabel const isBreakdown = payload.breakdowns.length && !alphabetIds.includes(key as 'A'); const serieName = isBreakdown ? key : getEventLegend(payload.event); const data = payload.chartType === 'area' || payload.chartType === 'linear' || payload.chartType === 'histogram' || payload.chartType === 'metric' || payload.chartType === 'pie' || payload.chartType === 'bar' ? fillEmptySpotsInTimeline( series[key] ?? [], payload.interval, payload.startDate, payload.endDate ).map((item) => { return { label: serieName, count: item.count ? round(item.count) : null, date: new Date(item.date).toISOString(), }; }) : (series[key] ?? []).map((item) => ({ label: item.label, count: item.count ? round(item.count) : null, date: new Date(item.date).toISOString(), })); return { name: serieName, event: payload.event, data, }; }); } export function getDatesFromRange(range: IChartRange) { if (range === '30min' || range === 'lastHour') { const minutes = range === '30min' ? 30 : 60; const startDate = subMinutes(new Date(), minutes).toUTCString(); const endDate = new Date().toUTCString(); return { startDate, endDate, }; } if (range === 'today') { const startDate = startOfDay(new Date()); const endDate = endOfDay(new Date()); return { startDate: startDate.toUTCString(), endDate: endDate.toUTCString(), }; } if (range === '7d') { const startDate = subDays(new Date(), 7).toUTCString(); const endDate = new Date().toUTCString(); return { startDate, endDate, }; } if (range === '30d') { const startDate = subDays(new Date(), 30).toUTCString(); const endDate = new Date().toUTCString(); return { startDate, endDate, }; } if (range === 'monthToDate') { const startDate = startOfMonth(new Date()).toUTCString(); const endDate = new Date().toUTCString(); return { startDate, endDate, }; } if (range === 'lastMonth') { const month = subMonths(new Date(), 1); const startDate = startOfMonth(month).toUTCString(); const endDate = endOfDay(month).toUTCString(); return { startDate, endDate, }; } if (range === 'yearToDate') { const startDate = startOfYear(new Date()).toUTCString(); const endDate = new Date().toUTCString(); return { startDate, endDate, }; } if (range === 'lastYear') { const year = subYears(new Date(), 1); const startDate = startOfYear(year).toUTCString(); const endDate = endOfYear(year).toUTCString(); return { startDate, endDate, }; } console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); console.log('-------------------------------'); return { startDate: subDays(new Date(), 30).toISOString(), endDate: new Date().toISOString(), }; } export function getChartStartEndDate({ startDate, endDate, range, }: Pick) { return startDate && endDate ? { startDate: startDate, endDate: endDate } : getDatesFromRange(range); } export function getChartPrevStartEndDate({ startDate, endDate, range, }: { startDate: string; endDate: string; range: IChartRange; }) { const diff = new Date(endDate).getTime() - new Date(startDate).getTime(); return { startDate: new Date(new Date(startDate).getTime() - diff).toISOString(), endDate: new Date(new Date(endDate).getTime() - diff).toISOString(), }; } const ONE_DAY_IN_SECONDS = 60 * 60 * 24; export async function getFunnelData({ projectId, startDate, endDate, ...payload }: IChartInput) { if (!startDate || !endDate) { throw new Error('startDate and endDate are required'); } if (payload.events.length === 0) { return { totalSessions: 0, steps: [], }; } const funnels = payload.events.map((event) => { const { sb, getWhere } = createSqlBuilder(); sb.where = getEventFiltersWhereClause(event.filters); sb.where.name = `name = ${escape(event.name)}`; return getWhere().replace('WHERE ', ''); }); const innerSql = `SELECT session_id, windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level FROM events WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(startDate)}' AND created_at <= '${formatClickhouseDate(endDate)}' AND name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) GROUP BY session_id`; const sql = `SELECT level, count() AS count FROM (${innerSql}) GROUP BY level ORDER BY level DESC`; const [funnelRes, sessionRes] = await Promise.all([ chQuery<{ level: number; count: number }>(sql), chQuery<{ count: number }>( `SELECT count(name) as count FROM events WHERE project_id = ${escape(projectId)} AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')` ), ]); if (funnelRes[0]?.level !== payload.events.length) { funnelRes.unshift({ level: payload.events.length, count: 0, }); } const totalSessions = sessionRes[0]?.count ?? 0; const filledFunnelRes = funnelRes.reduce( (acc, item, index) => { const diff = index !== 0 ? (acc[acc.length - 1]?.level ?? 0) - item.level : 1; if (diff > 1) { acc.push( ...reverse( repeat({}, diff - 1).map((_, index) => ({ count: acc[acc.length - 1]?.count ?? 0, level: item.level + index + 1, })) ) ); } return [ ...acc, { count: item.count + (acc[acc.length - 1]?.count ?? 0), level: item.level, }, ]; }, [] as typeof funnelRes ); const steps = reverse(filledFunnelRes) .filter((item) => item.level !== 0) .reduce( (acc, item, index, list) => { const prev = list[index - 1] ?? { count: totalSessions }; const event = payload.events[item.level - 1]!; return [ ...acc, { event: { ...event, displayName: event.displayName ?? event.name, }, count: item.count, percent: (item.count / totalSessions) * 100, dropoffCount: prev.count - item.count, dropoffPercent: 100 - (item.count / prev.count) * 100, previousCount: prev.count, }, ]; }, [] as { event: IChartEvent & { displayName: string }; count: number; percent: number; dropoffCount: number; dropoffPercent: number; previousCount: number; }[] ); return { totalSessions, steps, }; } export async function getFunnelStep({ projectId, startDate, endDate, step, ...payload }: IChartInput & { step: number; }) { if (!startDate || !endDate) { throw new Error('startDate and endDate are required'); } if (payload.events.length === 0) { throw new Error('no events selected'); } const funnels = payload.events.map((event) => { const { sb, getWhere } = createSqlBuilder(); sb.where = getEventFiltersWhereClause(event.filters); sb.where.name = `name = ${escape(event.name)}`; return getWhere().replace('WHERE ', ''); }); const innerSql = `SELECT session_id, windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level FROM events WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(startDate)}' AND created_at <= '${formatClickhouseDate(endDate)}' AND name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) GROUP BY session_id`; const profileIdsQuery = `WITH sessions AS (${innerSql}) SELECT DISTINCT e.profile_id as id FROM sessions s JOIN events e ON s.session_id = e.session_id WHERE s.level = ${step} AND e.project_id = ${escape(projectId)} AND e.created_at >= '${formatClickhouseDate(startDate)}' AND e.created_at <= '${formatClickhouseDate(endDate)}' AND name IN (${payload.events.map((event) => escape(event.name)).join(', ')}) ORDER BY e.created_at DESC LIMIT 500 `; const res = await chQuery<{ id: string; }>(profileIdsQuery); return getProfiles({ ids: res.map((r) => r.id) }); } export async function getSeriesFromEvents(input: IChartInput) { const { startDate, endDate } = input.startDate && input.endDate ? { startDate: input.startDate, endDate: input.endDate, } : getDatesFromRange(input.range); const series = ( await Promise.all( input.events.map(async (event) => getChartData({ ...input, startDate, endDate, event, }) ) ) ).flat(); try { return withFormula(input, series); } catch (e) { return series; } }