diff --git a/packages/common/index.ts b/packages/common/index.ts index 081f50fb..b30cb315 100644 --- a/packages/common/index.ts +++ b/packages/common/index.ts @@ -6,3 +6,4 @@ export * from './src/names'; export * from './src/string'; export * from './src/math'; export * from './src/slug'; +export * from './src/fill-series'; diff --git a/packages/common/package.json b/packages/common/package.json index 45b35728..d682c25a 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -8,6 +8,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "date-fns": "^3.3.1", "mathjs": "^12.3.2", "ramda": "^0.29.1", "slugify": "^1.6.6", @@ -15,6 +16,7 @@ "unique-names-generator": "^4.7.1" }, "devDependencies": { + "@openpanel/validation": "workspace:*", "@openpanel/eslint-config": "workspace:*", "@openpanel/prettier-config": "workspace:*", "@openpanel/tsconfig": "workspace:*", diff --git a/packages/common/src/fill-series.ts b/packages/common/src/fill-series.ts new file mode 100644 index 00000000..0a7c6f19 --- /dev/null +++ b/packages/common/src/fill-series.ts @@ -0,0 +1,93 @@ +import { + addDays, + addHours, + addMinutes, + addMonths, + format, + parseISO, + startOfDay, + startOfHour, + startOfMinute, + startOfMonth, +} from 'date-fns'; + +import type { IInterval } from '@openpanel/validation'; + +// Define the data structure +interface DataEntry { + label: string; + count: number | null; + date: string; +} + +// Function to round down the date to the nearest interval +function roundDate(date: Date, interval: IInterval): Date { + switch (interval) { + case 'minute': + return startOfMinute(date); + case 'hour': + return startOfHour(date); + case 'day': + return startOfDay(date); + case 'month': + return startOfMonth(date); + default: + return startOfMinute(date); + } +} + +// Function to complete the timeline for each label +export function completeTimeline( + data: DataEntry[], + _startDate: string, + _endDate: string, + interval: IInterval +) { + const startDate = parseISO(_startDate); + const endDate = parseISO(_endDate); + // Group data by label + const labelsMap = new Map>(); + data.forEach((entry) => { + const roundedDate = roundDate(parseISO(entry.date), interval); + const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss'); + + if (!labelsMap.has(entry.label)) { + labelsMap.set(entry.label, new Map()); + } + const labelData = labelsMap.get(entry.label); + labelData?.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0)); + }); + + // Complete the timeline for each label + const result: Record = {}; + labelsMap.forEach((counts, label) => { + let currentDate = roundDate(startDate, interval); + result[label] = []; + while (currentDate <= endDate) { + const dateKey = format(currentDate, 'yyyy-MM-dd HH:mm:ss'); + result[label]!.push({ + label: label, + date: dateKey, + count: counts.get(dateKey) || 0, + }); + + // Increment the current date based on the interval + switch (interval) { + case 'minute': + currentDate = addMinutes(currentDate, 1); + break; + case 'hour': + currentDate = addHours(currentDate, 1); + break; + case 'day': + currentDate = addDays(currentDate, 1); + break; + case 'month': + currentDate = addMonths(currentDate, 1); + break; + } + } + }); + + return result; +} diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts index be56c933..4f0635d7 100644 --- a/packages/db/src/services/chart.service.ts +++ b/packages/db/src/services/chart.service.ts @@ -23,35 +23,29 @@ export function getChartSql({ sb.where = getEventFiltersWhereClause(event.filters); sb.where.projectId = `project_id = ${escape(projectId)}`; - let labelValue = escape('*'); if (event.name !== '*') { - labelValue = `${escape(event.name)}`; - sb.select.label = `${labelValue} as label`; - sb.where.eventName = `name = ${labelValue}`; + sb.select.label = `${escape(event.name)} as label`; + sb.where.eventName = `name = ${escape(event.name)}`; } else { - sb.select.label = `${labelValue} as label`; + sb.select.label = `'*' as label`; } sb.select.count = `count(*) as count`; switch (interval) { case 'minute': { sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; - sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`; break; } case 'hour': { sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; - sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`; break; } case 'day': { sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; - sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`; break; } case 'month': { sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; - sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`; break; } } diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index 44a4d9f3..94b68127 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -17,7 +17,7 @@ import * as mathjs from 'mathjs'; import { repeat, reverse, sort } from 'ramda'; import { escape } from 'sqlstring'; -import { round } from '@openpanel/common'; +import { completeTimeline, round } from '@openpanel/common'; import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants'; import { chQuery, @@ -133,88 +133,59 @@ const toDynamicISODateWithTZ = ( // It will be converted to the correct timezone on the client return date.replace(' ', 'T'); } - return `${date}T00:00:00`; + return `${date}T00:00:00Z`; }; export async function getChartData(payload: IGetChartDataInput) { - let result = await chQuery(getChartSql(payload)); - - if (result.length === 0 && payload.breakdowns.length > 0) { - result = await chQuery( - getChartSql({ - ...payload, - breakdowns: [], - }) - ); + async function getSeries() { + const result = await chQuery(getChartSql(payload)); + if (result.length === 0 && payload.breakdowns.length > 0) { + return await chQuery( + getChartSql({ + ...payload, + breakdowns: [], + }) + ); + } + return result; } - // group by sql label - const series = result.reduce( - (acc, item) => { - // If we fill empty spots in the timeline (clickhouse) we wont get a label back - // take the event name as label - if (!item.label && item.count === 0) { - item.label = payload.event.name; - } + return getSeries() + .then((data) => + completeTimeline( + data.map((item) => { + const label = item.label?.trim() || NOT_SET_VALUE; - const label = item.label?.trim() || NOT_SET_VALUE; - // item.label can be null when using breakdowns on a property - // that doesn't exist on all events - if (label) { - if (acc[label]) { - acc[label]?.push(item); - } else { - acc[label] = [item]; - } - } - - return { - ...acc, - }; - }, - {} as Record - ); - - return Object.keys(series).map((key) => { - // If we have breakdowns, we want to use the breakdown key as the legend - // But only if it successfully broke it down, otherwise we use the getEventLabel - const isBreakdown = - payload.breakdowns.length && !alphabetIds.includes(key as 'A'); - const serieName = isBreakdown ? key : getEventLegend(payload.event); - const data = - payload.chartType === 'area' || - payload.chartType === 'linear' || - payload.chartType === 'histogram' || - payload.chartType === 'metric' || - payload.chartType === 'pie' || - payload.chartType === 'bar' - ? (series[key] ?? []).map((item) => { - return { - label: serieName, - count: item.count ? round(item.count) : null, - date: toDynamicISODateWithTZ( - item.date, - payload.startDate, - payload.interval - ), - }; - }) - : (series[key] ?? []).map((item) => ({ - label: item.label, + return { + ...item, count: item.count ? round(item.count) : null, + label, + }; + }), + payload.startDate, + payload.endDate, + payload.interval + ) + ) + .then((series) => { + return Object.keys(series).map((label) => { + const isBreakdown = + payload.breakdowns.length && !alphabetIds.includes(label as 'A'); + const serieLabel = isBreakdown ? label : getEventLegend(payload.event); + return { + name: serieLabel, + event: payload.event, + data: series[label]!.map((item) => ({ + ...item, date: toDynamicISODateWithTZ( item.date, payload.startDate, payload.interval ), - })); - - return { - name: serieName, - event: payload.event, - data, - }; - }); + })), + }; + }); + }); } export function getDatesFromRange(range: IChartRange) { diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index e77aeb52..c32736f0 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -184,7 +184,7 @@ export const chartRouter = createTRPCRouter({ }), // TODO: Make this private - chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => { + chart: publicProcedure.input(zChartInput).query(async ({ input }) => { const currentPeriod = getChartStartEndDate(input); const previousPeriod = getChartPrevStartEndDate({ range: input.range, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d35fcfaa..6a9b5c6e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -763,6 +763,9 @@ importers: packages/common: dependencies: + date-fns: + specifier: ^3.3.1 + version: 3.3.1 mathjs: specifier: ^12.3.2 version: 12.3.2 @@ -788,6 +791,9 @@ importers: '@openpanel/tsconfig': specifier: workspace:* version: link:../../tooling/typescript + '@openpanel/validation': + specifier: workspace:* + version: link:../validation '@types/node': specifier: ^18.16.0 version: 18.19.17