From 06fb6c4f3c4dd4cf823ec6b8d5a8ec7d9dbe7411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= Date: Fri, 21 Nov 2025 11:21:17 +0100 Subject: [PATCH] wip --- .vscode/settings.json | 2 +- apps/api/src/controllers/export.controller.ts | 57 +- apps/api/src/utils/ai-tools.ts | 2 +- apps/start/package.json | 1 - .../components/report-chart/common/empty.tsx | 4 +- .../common/report-table-toolbar.tsx | 46 + .../report-chart/common/report-table-utils.ts | 325 +++++ .../report-chart/common/report-table.tsx | 1042 ++++++++++++++--- .../report-chart/conversion/summary.tsx | 18 +- .../components/report-chart/funnel/index.tsx | 6 +- .../components/report-chart/line/chart.tsx | 96 +- .../report-chart/retention/index.tsx | 7 +- .../src/components/report/reportSlice.ts | 30 +- .../report/sidebar/ReportEvents.tsx | 152 +-- .../report/sidebar/ReportFormula.tsx | 24 - .../report/sidebar/ReportSeries.tsx | 413 +++++++ .../report/sidebar/ReportSidebar.tsx | 4 +- apps/start/src/components/ui/input-enter.tsx | 5 +- apps/start/src/modals/view-chart-users.tsx | 210 +++- packages/db/index.ts | 1 + packages/db/package.json | 1 + packages/db/src/clickhouse/query-builder.ts | 1 + packages/db/src/engine/compute.ts | 165 +++ packages/db/src/engine/fetch.ts | 151 +++ packages/db/src/engine/format.ts | 141 +++ packages/db/src/engine/index.ts | 77 ++ packages/db/src/engine/normalize.ts | 66 ++ packages/db/src/engine/plan.ts | 59 + packages/db/src/engine/types.ts | 85 ++ packages/db/src/services/chart.service.ts | 3 +- .../db/src/services/conversion.service.ts | 16 +- packages/db/src/services/funnel.service.ts | 22 +- packages/db/src/services/reports.service.ts | 2 +- .../trpc/src/routers/chart.helpers.test.ts | 544 --------- packages/trpc/src/routers/chart.helpers.ts | 889 -------------- packages/trpc/src/routers/chart.ts | 182 +-- packages/trpc/src/routers/report.ts | 4 +- packages/validation/src/index.ts | 43 +- packages/validation/src/types.validation.ts | 7 +- pnpm-lock.yaml | 13 +- 40 files changed, 2944 insertions(+), 1972 deletions(-) create mode 100644 apps/start/src/components/report-chart/common/report-table-toolbar.tsx create mode 100644 apps/start/src/components/report-chart/common/report-table-utils.ts delete mode 100644 apps/start/src/components/report/sidebar/ReportFormula.tsx create mode 100644 apps/start/src/components/report/sidebar/ReportSeries.tsx create mode 100644 packages/db/src/engine/compute.ts create mode 100644 packages/db/src/engine/fetch.ts create mode 100644 packages/db/src/engine/format.ts create mode 100644 packages/db/src/engine/index.ts create mode 100644 packages/db/src/engine/normalize.ts create mode 100644 packages/db/src/engine/plan.ts create mode 100644 packages/db/src/engine/types.ts delete mode 100644 packages/trpc/src/routers/chart.helpers.test.ts delete mode 100644 packages/trpc/src/routers/chart.helpers.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index e1eb0bea..95b7c494 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,7 +17,7 @@ "editor.defaultFormatter": "biomejs.biome" }, "[json]": { - "editor.defaultFormatter": "biomejs.biome" + "editor.defaultFormatter": "vscode.json-language-features" }, "editor.formatOnSave": true, "tailwindCSS.experimental.classRegex": [ diff --git a/apps/api/src/controllers/export.controller.ts b/apps/api/src/controllers/export.controller.ts index cbaa0608..850fd768 100644 --- a/apps/api/src/controllers/export.controller.ts +++ b/apps/api/src/controllers/export.controller.ts @@ -12,8 +12,12 @@ import { getEventsCountCached, getSettingsForProject, } from '@openpanel/db'; -import { getChart } from '@openpanel/trpc/src/routers/chart.helpers'; -import { zChartEvent, zChartInput } from '@openpanel/validation'; +import { ChartEngine } from '@openpanel/db'; +import { + zChartEvent, + zChartInput, + zChartInputBase, +} from '@openpanel/validation'; import { omit } from 'ramda'; async function getProjectId( @@ -139,7 +143,7 @@ export async function events( }); } -const chartSchemeFull = zChartInput +const chartSchemeFull = zChartInputBase .pick({ breakdowns: true, interval: true, @@ -151,14 +155,27 @@ const chartSchemeFull = zChartInput .extend({ project_id: z.string().optional(), projectId: z.string().optional(), - events: z.array( - z.object({ - name: z.string(), - filters: zChartEvent.shape.filters.optional(), - segment: zChartEvent.shape.segment.optional(), - property: zChartEvent.shape.property.optional(), - }), - ), + series: z + .array( + z.object({ + name: z.string(), + filters: zChartEvent.shape.filters.optional(), + segment: zChartEvent.shape.segment.optional(), + property: zChartEvent.shape.property.optional(), + }), + ) + .optional(), + // Backward compatibility - events will be migrated to series via preprocessing + events: z + .array( + z.object({ + name: z.string(), + filters: zChartEvent.shape.filters.optional(), + segment: zChartEvent.shape.segment.optional(), + property: zChartEvent.shape.property.optional(), + }), + ) + .optional(), }); export async function charts( @@ -179,9 +196,17 @@ export async function charts( const projectId = await getProjectId(request, reply); const { timezone } = await getSettingsForProject(projectId); - const { events, ...rest } = query.data; + const { events, series, ...rest } = query.data; - return getChart({ + // Use series if available, otherwise fall back to events (backward compat) + const eventSeries = (series ?? events ?? []).map((event: any) => ({ + ...event, + type: event.type ?? 'event', + segment: event.segment ?? 'event', + filters: event.filters ?? [], + })); + + return ChartEngine.execute({ ...rest, startDate: rest.startDate ? DateTime.fromISO(rest.startDate) @@ -194,11 +219,7 @@ export async function charts( .toFormat('yyyy-MM-dd HH:mm:ss') : undefined, projectId, - events: events.map((event) => ({ - ...event, - segment: event.segment ?? 'event', - filters: event.filters ?? [], - })), + series: eventSeries, chartType: 'linear', metric: 'sum', }); diff --git a/apps/api/src/utils/ai-tools.ts b/apps/api/src/utils/ai-tools.ts index ffe91e1c..e261b551 100644 --- a/apps/api/src/utils/ai-tools.ts +++ b/apps/api/src/utils/ai-tools.ts @@ -7,8 +7,8 @@ import { ch, clix, } from '@openpanel/db'; +import { ChartEngine } from '@openpanel/db'; import { getCache } from '@openpanel/redis'; -import { getChart } from '@openpanel/trpc/src/routers/chart.helpers'; import { zChartInputAI } from '@openpanel/validation'; import { tool } from 'ai'; import { z } from 'zod'; diff --git a/apps/start/package.json b/apps/start/package.json index bc45a52f..ea949289 100644 --- a/apps/start/package.json +++ b/apps/start/package.json @@ -103,7 +103,6 @@ "lodash.throttle": "^4.1.1", "lottie-react": "^2.4.0", "lucide-react": "^0.476.0", - "mathjs": "^12.3.2", "mitt": "^3.0.1", "nuqs": "^2.5.2", "prisma-error-enum": "^0.1.3", diff --git a/apps/start/src/components/report-chart/common/empty.tsx b/apps/start/src/components/report-chart/common/empty.tsx index e5f33702..73341a7c 100644 --- a/apps/start/src/components/report-chart/common/empty.tsx +++ b/apps/start/src/components/report-chart/common/empty.tsx @@ -17,10 +17,10 @@ export function ReportChartEmpty({ }) { const { isEditMode, - report: { events }, + report: { series }, } = useReportChartContext(); - if (events.length === 0) { + if (!series || series.length === 0) { return (
diff --git a/apps/start/src/components/report-chart/common/report-table-toolbar.tsx b/apps/start/src/components/report-chart/common/report-table-toolbar.tsx new file mode 100644 index 00000000..a7bf83f0 --- /dev/null +++ b/apps/start/src/components/report-chart/common/report-table-toolbar.tsx @@ -0,0 +1,46 @@ +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { List, Rows3, Search, X } from 'lucide-react'; + +interface ReportTableToolbarProps { + grouped: boolean; + onToggleGrouped: () => void; + search: string; + onSearchChange: (value: string) => void; + onUnselectAll: () => void; +} + +export function ReportTableToolbar({ + grouped, + onToggleGrouped, + search, + onSearchChange, + onUnselectAll, +}: ReportTableToolbarProps) { + return ( +
+
+ + onSearchChange(e.target.value)} + className="pl-8" + /> +
+
+ + +
+
+ ); +} diff --git a/apps/start/src/components/report-chart/common/report-table-utils.ts b/apps/start/src/components/report-chart/common/report-table-utils.ts new file mode 100644 index 00000000..4a398080 --- /dev/null +++ b/apps/start/src/components/report-chart/common/report-table-utils.ts @@ -0,0 +1,325 @@ +import { getPropertyLabel } from '@/translations/properties'; +import type { IChartData } from '@/trpc/client'; + +export type TableRow = { + id: string; + serieName: string; + breakdownValues: string[]; + sum: number; + average: number; + min: number; + max: number; + dateValues: Record; // date -> count + originalSerie: IChartData['series'][0]; + // Group metadata for collapse functionality + groupKey?: string; // Unique key for the group this row belongs to + parentGroupKey?: string; // Key of parent group (for nested groups) + isSummaryRow?: boolean; // True if this is a summary row for a collapsed group +}; + +export type GroupedTableRow = TableRow & { + // For grouped mode, indicates which breakdown levels should show empty cells + breakdownDisplay: (string | null)[]; // null means show empty cell +}; + +/** + * Extract unique dates from all series + */ +function getUniqueDates(series: IChartData['series']): string[] { + const dateSet = new Set(); + series.forEach((serie) => { + serie.data.forEach((d) => { + dateSet.add(d.date); + }); + }); + return Array.from(dateSet).sort(); +} + +/** + * Get breakdown property names from series + * Breakdown values are in names.slice(1), so we need to infer the property names + * from the breakdowns array or from the series structure + */ +function getBreakdownPropertyNames( + series: IChartData['series'], + breakdowns: Array<{ name: string }>, +): string[] { + // If we have breakdowns from state, use those + if (breakdowns.length > 0) { + return breakdowns.map((b) => getPropertyLabel(b.name)); + } + + // Otherwise, infer from series names + // All series should have the same number of breakdown values + if (series.length === 0) return []; + const firstSerie = series[0]; + const breakdownCount = firstSerie.names.length - 1; + return Array.from({ length: breakdownCount }, (_, i) => `Breakdown ${i + 1}`); +} + +/** + * Transform series into flat table rows + */ +export function createFlatRows( + series: IChartData['series'], + dates: string[], +): TableRow[] { + return series.map((serie) => { + const dateValues: Record = {}; + dates.forEach((date) => { + const dataPoint = serie.data.find((d) => d.date === date); + dateValues[date] = dataPoint?.count ?? 0; + }); + + return { + id: serie.id, + serieName: serie.names[0] ?? '', + breakdownValues: serie.names.slice(1), + sum: serie.metrics.sum, + average: serie.metrics.average, + min: serie.metrics.min, + max: serie.metrics.max, + dateValues, + originalSerie: serie, + }; + }); +} + +/** + * Transform series into grouped table rows + * Groups rows hierarchically by breakdown values + */ +export function createGroupedRows( + series: IChartData['series'], + dates: string[], +): GroupedTableRow[] { + const flatRows = createFlatRows(series, dates); + + // Sort by sum descending + flatRows.sort((a, b) => b.sum - a.sum); + + // Group rows by breakdown values hierarchically + const grouped: GroupedTableRow[] = []; + const breakdownCount = flatRows[0]?.breakdownValues.length ?? 0; + + if (breakdownCount === 0) { + // No breakdowns, just return flat rows + return flatRows.map((row) => ({ + ...row, + breakdownDisplay: [], + })); + } + + // Group rows hierarchically by breakdown values + // We need to group by parent breakdowns first, then by child breakdowns + // This creates the nested structure shown in the user's example + + // First, group by first breakdown value + const groupsByFirstBreakdown = new Map(); + flatRows.forEach((row) => { + const firstBreakdown = row.breakdownValues[0] ?? ''; + if (!groupsByFirstBreakdown.has(firstBreakdown)) { + groupsByFirstBreakdown.set(firstBreakdown, []); + } + groupsByFirstBreakdown.get(firstBreakdown)!.push(row); + }); + + // Sort groups by sum of highest row in group + const sortedGroups = Array.from(groupsByFirstBreakdown.entries()).sort( + (a, b) => { + const aMax = Math.max(...a[1].map((r) => r.sum)); + const bMax = Math.max(...b[1].map((r) => r.sum)); + return bMax - aMax; + }, + ); + + // Process each group hierarchically + sortedGroups.forEach(([firstBreakdownValue, groupRows]) => { + // Within each first-breakdown group, sort by sum + groupRows.sort((a, b) => b.sum - a.sum); + + // Generate group key for this first-breakdown group + const groupKey = firstBreakdownValue; + + // For each row in the group + groupRows.forEach((row, index) => { + const breakdownDisplay: (string | null)[] = []; + + if (index === 0) { + // First row shows all breakdown values + breakdownDisplay.push(...row.breakdownValues); + } else { + // Subsequent rows: compare with first row in group + const firstRow = groupRows[0]!; + + for (let i = 0; i < row.breakdownValues.length; i++) { + // Show empty if this breakdown matches the first row at this position + if (i < firstRow.breakdownValues.length) { + if (row.breakdownValues[i] === firstRow.breakdownValues[i]) { + breakdownDisplay.push(null); + } else { + breakdownDisplay.push(row.breakdownValues[i]!); + } + } else { + breakdownDisplay.push(row.breakdownValues[i]!); + } + } + } + + grouped.push({ + ...row, + breakdownDisplay, + groupKey, + }); + }); + }); + + return grouped; +} + +/** + * Create a summary row for a collapsed group + */ +export function createSummaryRow( + groupRows: TableRow[], + groupKey: string, + breakdownCount: number, +): GroupedTableRow { + // Aggregate metrics from all rows in the group + const totalSum = groupRows.reduce((sum, row) => sum + row.sum, 0); + const totalAverage = + groupRows.reduce((sum, row) => sum + row.average, 0) / groupRows.length; + const totalMin = Math.min(...groupRows.map((row) => row.min)); + const totalMax = Math.max(...groupRows.map((row) => row.max)); + + // Aggregate date values + const dateValues: Record = {}; + const allDates = new Set(); + groupRows.forEach((row) => { + Object.keys(row.dateValues).forEach((date) => { + allDates.add(date); + dateValues[date] = (dateValues[date] ?? 0) + row.dateValues[date]; + }); + }); + + // Get breakdown values from first row + const firstRow = groupRows[0]!; + const breakdownDisplay: (string | null)[] = []; + breakdownDisplay.push(firstRow.breakdownValues[0] ?? null); + // Fill remaining breakdowns with null (empty) + for (let i = 1; i < breakdownCount; i++) { + breakdownDisplay.push(null); + } + + return { + id: `summary-${groupKey}`, + serieName: firstRow.serieName, + breakdownValues: firstRow.breakdownValues, + sum: totalSum, + average: totalAverage, + min: totalMin, + max: totalMax, + dateValues, + originalSerie: firstRow.originalSerie, + groupKey, + isSummaryRow: true, + breakdownDisplay, + }; +} + +/** + * Reorder breakdowns by number of unique values (fewest first) + */ +function reorderBreakdownsByUniqueCount( + series: IChartData['series'], + breakdownPropertyNames: string[], +): { + reorderedNames: string[]; + reorderMap: number[]; // Maps new index -> old index + reverseMap: number[]; // Maps old index -> new index +} { + if (breakdownPropertyNames.length === 0 || series.length === 0) { + return { + reorderedNames: breakdownPropertyNames, + reorderMap: [], + reverseMap: [], + }; + } + + // Count unique values for each breakdown index + const uniqueCounts = breakdownPropertyNames.map((_, index) => { + const uniqueValues = new Set(); + series.forEach((serie) => { + const value = serie.names[index + 1]; // +1 because names[0] is serie name + if (value) { + uniqueValues.add(value); + } + }); + return { index, count: uniqueValues.size }; + }); + + // Sort by count (ascending - fewest first) + uniqueCounts.sort((a, b) => a.count - b.count); + + // Create reordered names and mapping + const reorderedNames = uniqueCounts.map( + (item) => breakdownPropertyNames[item.index]!, + ); + const reorderMap = uniqueCounts.map((item) => item.index); // new index -> old index + const reverseMap = new Array(breakdownPropertyNames.length); + reorderMap.forEach((oldIndex, newIndex) => { + reverseMap[oldIndex] = newIndex; + }); + + return { reorderedNames, reorderMap, reverseMap }; +} + +/** + * Transform chart data into table-ready format + */ +export function transformToTableData( + data: IChartData, + breakdowns: Array<{ name: string }>, + grouped: boolean, +): { + rows: TableRow[] | GroupedTableRow[]; + dates: string[]; + breakdownPropertyNames: string[]; +} { + const dates = getUniqueDates(data.series); + const originalBreakdownPropertyNames = getBreakdownPropertyNames( + data.series, + breakdowns, + ); + + // Reorder breakdowns by unique count (fewest first) + const { reorderedNames: breakdownPropertyNames, reorderMap } = + reorderBreakdownsByUniqueCount(data.series, originalBreakdownPropertyNames); + + // Reorder breakdown values in series before creating rows + const reorderedSeries = data.series.map((serie) => { + const reorderedNames = [ + serie.names[0], // Keep serie name first + ...reorderMap.map((oldIndex) => serie.names[oldIndex + 1] ?? ''), // Reorder breakdown values + ]; + return { + ...serie, + names: reorderedNames, + }; + }); + + const rows = grouped + ? createGroupedRows(reorderedSeries, dates) + : createFlatRows(reorderedSeries, dates); + + // Sort flat rows by sum descending + if (!grouped) { + (rows as TableRow[]).sort((a, b) => b.sum - a.sum); + } + + return { + rows, + dates, + breakdownPropertyNames, + }; +} diff --git a/apps/start/src/components/report-chart/common/report-table.tsx b/apps/start/src/components/report-chart/common/report-table.tsx index a4d21892..59adc4c0 100644 --- a/apps/start/src/components/report-chart/common/report-table.tsx +++ b/apps/start/src/components/report-chart/common/report-table.tsx @@ -1,6 +1,3 @@ -import { Pagination, usePagination } from '@/components/pagination'; -import { Stats, StatsCard } from '@/components/stats'; -import { Badge } from '@/components/ui/badge'; import { Checkbox } from '@/components/ui/checkbox'; import { Table, @@ -8,35 +5,73 @@ import { TableCell, TableHead, TableHeader, - TableRow, + TableRow as UITableRow, } from '@/components/ui/table'; -import { Tooltiper } from '@/components/ui/tooltip'; import { useFormatDateInterval } from '@/hooks/use-format-date-interval'; import { useNumber } from '@/hooks/use-numer-formatter'; import { useSelector } from '@/redux'; -import { getPropertyLabel } from '@/translations/properties'; import type { IChartData } from '@/trpc/client'; +import { cn } from '@/utils/cn'; import { getChartColor } from '@/utils/theme'; +import type { ColumnDef } from '@tanstack/react-table'; +import { + type SortingState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getSortedRowModel, + useReactTable, +} from '@tanstack/react-table'; +import { + type VirtualItem, + useWindowVirtualizer, +} from '@tanstack/react-virtual'; +import throttle from 'lodash.throttle'; +import { ChevronDown, ChevronRight } from 'lucide-react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import type * as React from 'react'; -import { logDependencies } from 'mathjs'; -import { PreviousDiffIndicator } from './previous-diff-indicator'; +import { ReportTableToolbar } from './report-table-toolbar'; +import { + type GroupedTableRow, + type TableRow, + createSummaryRow, + transformToTableData, +} from './report-table-utils'; import { SerieName } from './serie-name'; +declare module '@tanstack/react-table' { + interface ColumnMeta { + pinned?: 'left' | 'right'; + isBreakdown?: boolean; + breakdownIndex?: number; + } +} + interface ReportTableProps { data: IChartData; - visibleSeries: IChartData['series']; + visibleSeries: IChartData['series'] | string[]; setVisibleSeries: React.Dispatch>; } -const ROWS_LIMIT = 50; +const DEFAULT_COLUMN_WIDTH = 150; +const ROW_HEIGHT = 48; // h-12 export function ReportTable({ data, visibleSeries, setVisibleSeries, }: ReportTableProps) { - const { setPage, paginate, page } = usePagination(ROWS_LIMIT); + const [grouped, setGrouped] = useState(true); + const [collapsedGroups, setCollapsedGroups] = useState>( + new Set(), + ); + const [sorting, setSorting] = useState([]); + const [globalFilter, setGlobalFilter] = useState(''); + const [columnSizing, setColumnSizing] = useState>({}); + const isResizingRef = useRef(false); + const parentRef = useRef(null); + const [scrollMargin, setScrollMargin] = useState(0); const number = useNumber(); const interval = useSelector((state) => state.report.interval); const breakdowns = useSelector((state) => state.report.breakdowns); @@ -45,149 +80,870 @@ export function ReportTable({ short: true, }); - function handleChange(name: string, checked: boolean) { - setVisibleSeries((prev) => { - if (checked) { - return [...prev, name]; + // Transform data to table format + const { + rows: rawRows, + dates, + breakdownPropertyNames, + } = useMemo( + () => transformToTableData(data, breakdowns, grouped), + [data, breakdowns, grouped], + ); + + // Filter rows based on collapsed groups and create summary rows + const rows = useMemo(() => { + if (!grouped || collapsedGroups.size === 0) { + return rawRows; + } + + const processedRows: (TableRow | GroupedTableRow)[] = []; + const groupedRows = rawRows as GroupedTableRow[]; + + // Group rows by their groupKey + const rowsByGroup = new Map(); + groupedRows.forEach((row) => { + if (row.groupKey) { + if (!rowsByGroup.has(row.groupKey)) { + rowsByGroup.set(row.groupKey, []); + } + rowsByGroup.get(row.groupKey)!.push(row); + } else { + // Rows without groupKey go directly to processed + processedRows.push(row); } - return prev.filter((item) => item !== name); }); + + // Process each group + rowsByGroup.forEach((groupRows, groupKey) => { + if (collapsedGroups.has(groupKey)) { + // Group is collapsed - show summary row + const summaryRow = createSummaryRow( + groupRows, + groupKey, + breakdownPropertyNames.length, + ); + processedRows.push(summaryRow); + } else { + // Group is expanded - show all rows + processedRows.push(...groupRows); + } + }); + + return processedRows; + }, [rawRows, collapsedGroups, grouped, breakdownPropertyNames.length]); + + // Filter rows based on global search and apply sorting + const filteredRows = useMemo(() => { + let result = rows; + + // Apply search filter + if (globalFilter.trim()) { + const searchLower = globalFilter.toLowerCase(); + result = rows.filter((row) => { + // Search in serie name + if (row.serieName.toLowerCase().includes(searchLower)) return true; + + // Search in breakdown values + if ( + row.breakdownValues.some((val) => + val?.toLowerCase().includes(searchLower), + ) + ) { + return true; + } + + // Search in metric values + const metrics = ['sum', 'average', 'min', 'max'] as const; + if ( + metrics.some((metric) => + String(row[metric]).toLowerCase().includes(searchLower), + ) + ) { + return true; + } + + // Search in date values + if ( + Object.values(row.dateValues).some((val) => + String(val).toLowerCase().includes(searchLower), + ) + ) { + return true; + } + + return false; + }); + } + + // Apply sorting - if grouped, sort within each group + if (grouped && sorting.length > 0 && result.length > 0) { + const groupedRows = result as GroupedTableRow[]; + + // Group rows by their groupKey + const rowsByGroup = new Map(); + const ungroupedRows: GroupedTableRow[] = []; + + groupedRows.forEach((row) => { + if (row.groupKey) { + if (!rowsByGroup.has(row.groupKey)) { + rowsByGroup.set(row.groupKey, []); + } + rowsByGroup.get(row.groupKey)!.push(row); + } else { + ungroupedRows.push(row); + } + }); + + // Sort function based on current sort state + const sortFn = (a: GroupedTableRow, b: GroupedTableRow) => { + for (const sort of sorting) { + const { id, desc } = sort; + let aValue: any; + let bValue: any; + + if (id === 'serie-name') { + aValue = a.serieName; + bValue = b.serieName; + } else if (id.startsWith('breakdown-')) { + const index = Number.parseInt(id.replace('breakdown-', ''), 10); + aValue = a.breakdownValues[index] ?? ''; + bValue = b.breakdownValues[index] ?? ''; + } else if (id.startsWith('metric-')) { + const metric = id.replace('metric-', '') as keyof TableRow; + aValue = a[metric]; + bValue = b[metric]; + } else if (id.startsWith('date-')) { + const date = id.replace('date-', ''); + aValue = a.dateValues[date] ?? 0; + bValue = b.dateValues[date] ?? 0; + } else { + continue; + } + + // Compare values + if (aValue < bValue) return desc ? 1 : -1; + if (aValue > bValue) return desc ? -1 : 1; + } + return 0; + }; + + // Sort groups themselves by their first row's sort value + const groupsArray = Array.from(rowsByGroup.entries()); + groupsArray.sort((a, b) => { + const aFirst = a[1][0]; + const bFirst = b[1][0]; + if (!aFirst || !bFirst) return 0; + return sortFn(aFirst, bFirst); + }); + + // Rebuild result with sorted groups + const finalResult: GroupedTableRow[] = []; + groupsArray.forEach(([, groupRows]) => { + const sorted = [...groupRows].sort(sortFn); + finalResult.push(...sorted); + }); + finalResult.push(...ungroupedRows.sort(sortFn)); + + return finalResult; + } + + return result; + }, [rows, globalFilter, grouped, sorting]); + + // Calculate min/max values for color visualization + const { metricRanges, dateRanges } = useMemo(() => { + const metricRanges: Record = { + sum: { min: Number.POSITIVE_INFINITY, max: Number.NEGATIVE_INFINITY }, + average: { + min: Number.POSITIVE_INFINITY, + max: Number.NEGATIVE_INFINITY, + }, + min: { min: Number.POSITIVE_INFINITY, max: Number.NEGATIVE_INFINITY }, + max: { min: Number.POSITIVE_INFINITY, max: Number.NEGATIVE_INFINITY }, + }; + + const dateRanges: Record = {}; + dates.forEach((date) => { + dateRanges[date] = { + min: Number.POSITIVE_INFINITY, + max: Number.NEGATIVE_INFINITY, + }; + }); + + rows.forEach((row) => { + // Calculate metric ranges + Object.keys(metricRanges).forEach((key) => { + const value = row[key as keyof typeof row] as number; + if (typeof value === 'number') { + metricRanges[key]!.min = Math.min(metricRanges[key]!.min, value); + metricRanges[key]!.max = Math.max(metricRanges[key]!.max, value); + } + }); + + // Calculate date ranges + dates.forEach((date) => { + const value = row.dateValues[date] ?? 0; + if (!dateRanges[date]) { + dateRanges[date] = { + min: Number.POSITIVE_INFINITY, + max: Number.NEGATIVE_INFINITY, + }; + } + dateRanges[date]!.min = Math.min(dateRanges[date]!.min, value); + dateRanges[date]!.max = Math.max(dateRanges[date]!.max, value); + }); + }); + + return { metricRanges, dateRanges }; + }, [rows, dates]); + + // Helper to get background color and opacity for a value + const getCellBackground = ( + value: number, + min: number, + max: number, + ): { opacity: number; className: string } => { + if (value === 0 || max === min) { + return { opacity: 0, className: '' }; + } + + const percentage = (value - min) / (max - min); + const opacity = Math.max(0.05, Math.min(1, percentage)); + + return { + opacity, + className: 'bg-highlight dark:bg-emerald-700', + }; + }; + + // Normalize visibleSeries to string array + const visibleSeriesIds = useMemo(() => { + if (visibleSeries.length === 0) return []; + if (typeof visibleSeries[0] === 'string') { + return visibleSeries as string[]; + } + return (visibleSeries as IChartData['series']).map((s) => s.id); + }, [visibleSeries]); + + // Get serie index for color + const getSerieIndex = (serieId: string): number => { + return data.series.findIndex((s) => s.id === serieId); + }; + + // Toggle serie visibility + const toggleSerieVisibility = (serieId: string) => { + setVisibleSeries((prev) => { + if (prev.includes(serieId)) { + return prev.filter((id) => id !== serieId); + } + return [...prev, serieId]; + }); + }; + + // Toggle group collapse + const toggleGroupCollapse = (groupKey: string) => { + setCollapsedGroups((prev) => { + const next = new Set(prev); + if (next.has(groupKey)) { + next.delete(groupKey); + } else { + next.add(groupKey); + } + return next; + }); + }; + + // Define columns + const columns = useMemo[]>(() => { + const cols: ColumnDef[] = []; + + // Serie name column (pinned left) with checkbox + cols.push({ + id: 'serie-name', + header: 'Serie', + accessorKey: 'serieName', + enableSorting: true, + size: DEFAULT_COLUMN_WIDTH, + meta: { + pinned: 'left', + }, + cell: ({ row }) => { + const serieName = row.original.serieName; + const serieId = row.original.originalSerie.id; + const isVisible = visibleSeriesIds.includes(serieId); + const serieIndex = getSerieIndex(serieId); + const color = getChartColor(serieIndex); + + return ( +
+ toggleSerieVisibility(serieId)} + style={{ + borderColor: color, + backgroundColor: isVisible ? color : 'transparent', + }} + className="h-4 w-4 shrink-0" + /> + +
+ ); + }, + }); + + // Breakdown columns (pinned left, collapsible) + breakdownPropertyNames.forEach((propertyName, index) => { + const isLastBreakdown = index === breakdownPropertyNames.length - 1; + const isCollapsible = grouped && !isLastBreakdown; + + cols.push({ + id: `breakdown-${index}`, + enableSorting: true, + enableResizing: true, + size: columnSizing[`breakdown-${index}`] ?? DEFAULT_COLUMN_WIDTH, + minSize: 100, + maxSize: 500, + accessorFn: (row) => { + if ('breakdownDisplay' in row && grouped) { + return row.breakdownDisplay[index] ?? ''; + } + return row.breakdownValues[index] ?? ''; + }, + header: ({ column }) => { + if (!isCollapsible) { + return propertyName; + } + + // Find all unique group keys for this breakdown level + const groupKeys = new Set(); + (rawRows as GroupedTableRow[]).forEach((row) => { + if (row.groupKey) { + groupKeys.add(row.groupKey); + } + }); + + // Check if all groups at this level are collapsed + const allCollapsed = Array.from(groupKeys).every((key) => + collapsedGroups.has(key), + ); + + return ( +
{ + // Toggle all groups at this breakdown level + groupKeys.forEach((key) => toggleGroupCollapse(key)); + }} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + groupKeys.forEach((key) => toggleGroupCollapse(key)); + } + }} + role="button" + tabIndex={0} + > + {allCollapsed ? ( + + ) : ( + + )} + {propertyName} +
+ ); + }, + meta: { + pinned: 'left', + isBreakdown: true, + breakdownIndex: index, + }, + cell: ({ row }) => { + const original = row.original; + let value: string | null; + + if ('breakdownDisplay' in original && grouped) { + value = original.breakdownDisplay[index] ?? null; + } else { + value = original.breakdownValues[index] ?? null; + } + + const isSummary = original.isSummaryRow ?? false; + + return ( + + {value || ''} + + ); + }, + }); + }); + + // Metric columns + const metrics = [ + { key: 'sum', label: 'Sum' }, + { key: 'average', label: 'Average' }, + { key: 'min', label: 'Min' }, + { key: 'max', label: 'Max' }, + ] as const; + + metrics.forEach((metric) => { + cols.push({ + id: `metric-${metric.key}`, + header: metric.label, + accessorKey: metric.key, + enableSorting: true, + size: 100, + cell: ({ row }) => { + const value = row.original[metric.key]; + const isSummary = row.original.isSummaryRow ?? false; + const range = metricRanges[metric.key]; + const { opacity, className } = range + ? getCellBackground(value, range.min, range.max) + : { opacity: 0, className: '' }; + + return ( +
+
+
0.7 && + 'text-white [text-shadow:_0_0_3px_rgb(0_0_0_/_20%)]', + )} + > + {number.format(value)} +
+
+ ); + }, + }); + }); + + // Date columns + dates.forEach((date) => { + cols.push({ + id: `date-${date}`, + header: formatDate(date), + accessorFn: (row) => row.dateValues[date] ?? 0, + enableSorting: true, + size: 100, + cell: ({ row }) => { + const value = row.original.dateValues[date] ?? 0; + const isSummary = row.original.isSummaryRow ?? false; + const range = dateRanges[date]; + const { opacity, className } = range + ? getCellBackground(value, range.min, range.max) + : { opacity: 0, className: '' }; + + return ( +
+
+
0.7 && + 'text-white [text-shadow:_0_0_3px_rgb(0_0_0_/_20%)]', + )} + > + {number.format(value)} +
+
+ ); + }, + }); + }); + + return cols; + }, [ + breakdownPropertyNames, + dates, + formatDate, + number, + grouped, + visibleSeriesIds, + collapsedGroups, + rawRows, + metricRanges, + dateRanges, + columnSizing, + ]); + + const table = useReactTable({ + data: filteredRows, + columns, + getCoreRowModel: getCoreRowModel(), + getSortedRowModel: grouped ? getCoreRowModel() : getSortedRowModel(), // Disable TanStack sorting when grouped + getFilteredRowModel: getFilteredRowModel(), + filterFns: { + isWithinRange: () => true, + }, + enableColumnResizing: true, + columnResizeMode: 'onChange', + state: { + sorting, + columnSizing, + }, + onSortingChange: setSorting, + onColumnSizingChange: setColumnSizing, + globalFilterFn: () => true, // We handle filtering manually + manualSorting: grouped, // Manual sorting when grouped + }); + + // Virtualization setup + useEffect(() => { + const updateScrollMargin = throttle(() => { + if (parentRef.current) { + setScrollMargin( + parentRef.current.getBoundingClientRect().top + window.scrollY, + ); + } + }, 500); + + updateScrollMargin(); + window.addEventListener('resize', updateScrollMargin); + + return () => { + window.removeEventListener('resize', updateScrollMargin); + }; + }, []); + + // Handle global mouseup to reset resize flag + useEffect(() => { + const handleMouseUp = () => { + if (isResizingRef.current) { + // Small delay to ensure resize handlers complete + setTimeout(() => { + isResizingRef.current = false; + }, 100); + } + }; + + window.addEventListener('mouseup', handleMouseUp); + window.addEventListener('touchend', handleMouseUp); + + return () => { + window.removeEventListener('mouseup', handleMouseUp); + window.removeEventListener('touchend', handleMouseUp); + }; + }, []); + + const virtualizer = useWindowVirtualizer({ + count: filteredRows.length, + estimateSize: () => ROW_HEIGHT, + overscan: 10, + scrollMargin, + }); + + const virtualRows = virtualizer.getVirtualItems(); + + // Get visible columns in order + const headerColumns = table + .getAllLeafColumns() + .filter((col) => table.getState().columnVisibility[col.id] !== false); + + // Get pinned columns + const leftPinnedColumns = table + .getAllColumns() + .filter((col) => col.columnDef.meta?.pinned === 'left') + .filter((col): col is NonNullable => col !== undefined); + const rightPinnedColumns = table + .getAllColumns() + .filter((col) => col.columnDef.meta?.pinned === 'right') + .filter((col): col is NonNullable => col !== undefined); + + // Helper to get pinning styles + const getPinningStyles = ( + column: ReturnType | undefined, + ) => { + if (!column) return {}; + const isPinned = column.columnDef.meta?.pinned; + if (!isPinned) return {}; + + const pinnedColumns = + isPinned === 'left' ? leftPinnedColumns : rightPinnedColumns; + const columnIndex = pinnedColumns.findIndex((c) => c.id === column.id); + const isLastPinned = + columnIndex === pinnedColumns.length - 1 && isPinned === 'left'; + const isFirstRightPinned = columnIndex === 0 && isPinned === 'right'; + + let left = 0; + if (isPinned === 'left') { + for (let i = 0; i < columnIndex; i++) { + left += pinnedColumns[i]!.getSize(); + } + } + + return { + position: 'sticky' as const, + left: isPinned === 'left' ? `${left}px` : undefined, + right: isPinned === 'right' ? '0px' : undefined, + zIndex: 10, + backgroundColor: 'var(--card)', + boxShadow: isLastPinned + ? '-4px 0 4px -4px var(--border) inset' + : isFirstRightPinned + ? '4px 0 4px -4px var(--border) inset' + : undefined, + }; + }; + + if (rows.length === 0) { + return null; } return ( - <> - - - - - - -
- - - - {breakdowns.length === 0 && Name} - {breakdowns.map((breakdown) => ( - - {getPropertyLabel(breakdown.name)} - - ))} - - - - {paginate(data.series).map((serie, index) => { - const checked = !!visibleSeries.find( - (item) => item.id === serie.id, - ); +
+ setGrouped(!grouped)} + search={globalFilter} + onSearchChange={setGlobalFilter} + onUnselectAll={() => setVisibleSeries([])} + /> +
+
+ {/* Header */} +
`${h.getSize()}px`) + .join(' ') ?? '', + minWidth: 'fit-content', + }} + > + {table.getHeaderGroups()[0]?.headers.map((header) => { + const column = header.column; + const headerContent = column.columnDef.header; + const isBreakdown = column.columnDef.meta?.isBreakdown ?? false; + const pinningStyles = getPinningStyles(column); + const isMetricOrDate = + column.id.startsWith('metric-') || + column.id.startsWith('date-'); + + const canSort = column.getCanSort(); + const isSorted = column.getIsSorted(); + const canResize = column.getCanResize(); + const isPinned = column.columnDef.meta?.pinned === 'left'; return ( - - {serie.names.map((name, nameIndex) => { - return ( - -
- {nameIndex === 0 ? ( - <> - - handleChange(serie.id, !!checked) - } - style={ - checked - ? { - background: getChartColor(index), - borderColor: getChartColor(index), - } - : undefined - } - checked={checked} - /> - } - > - {name} - - - ) : ( - - )} -
- - ); - })} - +
{ + // Don't trigger sort if clicking on resize handle or if we just finished resizing + if ( + isResizingRef.current || + column.getIsResizing() || + (e.target as HTMLElement).closest( + '[data-resize-handle]', + ) + ) { + return; + } + column.toggleSorting(); + } + : undefined + } + onKeyDown={ + canSort + ? (e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + column.toggleSorting(); + } + } + : undefined + } + role={canSort ? 'button' : undefined} + tabIndex={canSort ? 0 : undefined} + > +
+ {header.isPlaceholder + ? null + : typeof headerContent === 'function' + ? flexRender(headerContent, header.getContext()) + : headerContent} + {canSort && ( + + {isSorted === 'asc' + ? '↑' + : isSorted === 'desc' + ? '↓' + : '⇅'} + + )} +
+ {canResize && isPinned && ( +
{ + e.stopPropagation(); + isResizingRef.current = true; + header.getResizeHandler()(e); + }} + onMouseUp={() => { + // Use setTimeout to allow the resize to complete before resetting + setTimeout(() => { + isResizingRef.current = false; + }, 0); + }} + onTouchStart={(e) => { + e.stopPropagation(); + isResizingRef.current = true; + header.getResizeHandler()(e); + }} + onTouchEnd={() => { + setTimeout(() => { + isResizingRef.current = false; + }, 0); + }} + className={cn( + 'absolute right-0 top-0 h-full w-1 cursor-col-resize touch-none select-none bg-transparent hover:bg-primary/50 transition-colors', + header.column.getIsResizing() && 'bg-primary', + )} + /> + )} +
); })} - -
-
- - - - Total - Average - {data.series[0]?.data.map((serie) => ( - - {formatDate(serie.date)} - - ))} - - - - {paginate(data.series).map((serie) => { - return ( - - -
- {number.format(serie.metrics.sum)} - -
- - -
- {number.format(serie.metrics.average)} - -
- + - {serie.data.map((item) => { - return ( - -
- {number.format(item.count)} - -
- - ); - })} - - ); - })} - -
+ {/* Virtualized Body */} +
+ {virtualRows.map((virtualRow) => { + const tableRow = table.getRowModel().rows[virtualRow.index]; + if (!tableRow) return null; + + return ( +
`${h.getSize()}px`) + .join(' ') ?? '', + minWidth: 'fit-content', + }} + className="border-b hover:bg-muted/30 transition-colors" + > + {table.getHeaderGroups()[0]?.headers.map((header) => { + const column = header.column; + const cell = tableRow + .getVisibleCells() + .find((c) => c.column.id === column.id); + if (!cell) return null; + + const isBreakdown = + column.columnDef.meta?.isBreakdown ?? false; + const pinningStyles = getPinningStyles(column); + const isMetricOrDate = + column.id.startsWith('metric-') || + column.id.startsWith('date-'); + + const canResize = column.getCanResize(); + const isPinned = column.columnDef.meta?.pinned === 'left'; + + return ( +
+ {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + {canResize && isPinned && ( +
{ + e.stopPropagation(); + isResizingRef.current = true; + header.getResizeHandler()(e); + }} + onMouseUp={() => { + setTimeout(() => { + isResizingRef.current = false; + }, 0); + }} + onTouchStart={(e) => { + e.stopPropagation(); + isResizingRef.current = true; + header.getResizeHandler()(e); + }} + onTouchEnd={() => { + setTimeout(() => { + isResizingRef.current = false; + }, 0); + }} + className={cn( + 'absolute right-0 top-0 h-full w-1 cursor-col-resize touch-none select-none bg-transparent hover:bg-primary/50 transition-colors', + column.getIsResizing() && 'bg-primary', + )} + /> + )} +
+ ); + })} +
+ ); + })} +
- - {/*
- -
*/} - +
); } diff --git a/apps/start/src/components/report-chart/conversion/summary.tsx b/apps/start/src/components/report-chart/conversion/summary.tsx index 683d8fc1..60dc97de 100644 --- a/apps/start/src/components/report-chart/conversion/summary.tsx +++ b/apps/start/src/components/report-chart/conversion/summary.tsx @@ -144,14 +144,16 @@ export function Summary({ data }: Props) { title="Flow" value={
- {report.events.map((event, index) => { - return ( -
- {index !== 0 && } - {event.name} -
- ); - })} + {report.series + .filter((item) => item.type === 'event') + .map((event, index) => { + return ( +
+ {index !== 0 && } + {event.name} +
+ ); + })}
} /> diff --git a/apps/start/src/components/report-chart/funnel/index.tsx b/apps/start/src/components/report-chart/funnel/index.tsx index e80063cc..fdffd671 100644 --- a/apps/start/src/components/report-chart/funnel/index.tsx +++ b/apps/start/src/components/report-chart/funnel/index.tsx @@ -14,7 +14,7 @@ import { Chart, Summary, Tables } from './chart'; export function ReportFunnelChart() { const { report: { - events, + series, range, projectId, funnelWindow, @@ -28,7 +28,7 @@ export function ReportFunnelChart() { } = useReportChartContext(); const input: IChartInput = { - events, + series, range, projectId, interval: 'day', @@ -44,7 +44,7 @@ export function ReportFunnelChart() { const trpc = useTRPC(); const res = useQuery( trpc.chart.funnel.queryOptions(input, { - enabled: !isLazyLoading && input.events.length > 0, + enabled: !isLazyLoading && input.series.length > 0, }), ); diff --git a/apps/start/src/components/report-chart/line/chart.tsx b/apps/start/src/components/report-chart/line/chart.tsx index ab3365e3..87b6e500 100644 --- a/apps/start/src/components/report-chart/line/chart.tsx +++ b/apps/start/src/components/report-chart/line/chart.tsx @@ -1,3 +1,9 @@ +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; import { useRechartDataModel } from '@/hooks/use-rechart-data-model'; import { useVisibleSeries } from '@/hooks/use-visible-series'; import { useTRPC } from '@/integrations/trpc/react'; @@ -5,17 +11,12 @@ import { pushModal } from '@/modals'; import type { IChartData } from '@/trpc/client'; import { cn } from '@/utils/cn'; import { getChartColor } from '@/utils/theme'; +import type { IChartEvent } from '@openpanel/validation'; import { useQuery } from '@tanstack/react-query'; import { isSameDay, isSameHour, isSameMonth, isSameWeek } from 'date-fns'; +import { BookmarkIcon, UsersIcon } from 'lucide-react'; import { last } from 'ramda'; import { useCallback, useState } from 'react'; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from '@/components/ui/dropdown-menu'; -import { UsersIcon, BookmarkIcon } from 'lucide-react'; import { CartesianGrid, ComposedChart, @@ -51,14 +52,20 @@ export function Chart({ data }: Props) { endDate, range, lineType, - events, + series: reportSeries, breakdowns, }, isEditMode, options: { hideXAxis, hideYAxis, maxDomain }, } = useReportChartContext(); - const [clickPosition, setClickPosition] = useState<{ x: number; y: number } | null>(null); - const [clickedData, setClickedData] = useState<{ date: string; serieId?: string } | null>(null); + const [clickPosition, setClickPosition] = useState<{ + x: number; + y: number; + } | null>(null); + const [clickedData, setClickedData] = useState<{ + date: string; + serieId?: string; + } | null>(null); const dataLength = data.series[0]?.data?.length || 0; const trpc = useTRPC(); const references = useQuery( @@ -144,9 +151,19 @@ export function Chart({ data }: Props) { const payload = e.activePayload[0].payload; const activeCoordinate = e.activeCoordinate; if (payload.date) { + // Find the first valid serie ID from activePayload (skip calcStrokeDasharray) + const validPayload = e.activePayload.find( + (p: any) => + p.dataKey && + p.dataKey !== 'calcStrokeDasharray' && + typeof p.dataKey === 'string' && + p.dataKey.includes(':count'), + ); + const serieId = validPayload?.dataKey?.toString().replace(':count', ''); + setClickedData({ date: payload.date, - serieId: e.activePayload[0].dataKey?.toString().replace(':count', ''), + serieId, }); setClickPosition({ x: activeCoordinate?.x ?? 0, @@ -157,36 +174,39 @@ export function Chart({ data }: Props) { }, []); const handleViewUsers = useCallback(() => { - if (!clickedData || !projectId || !startDate || !endDate) return; - - // Find the event for the clicked serie - const serie = series.find((s) => s.id === clickedData.serieId); - const event = events.find((e) => { - const normalized = 'type' in e ? e : { ...e, type: 'event' as const }; - if (normalized.type === 'event') { - return serie?.event.id === normalized.id || serie?.event.name === normalized.name; - } - return false; - }); + if (!clickedData || !projectId) return; - if (event) { - const normalized = 'type' in event ? event : { ...event, type: 'event' as const }; - if (normalized.type === 'event') { - pushModal('ViewChartUsers', { - projectId, - event: normalized, - date: clickedData.date, - breakdowns: breakdowns || [], - interval, - startDate, - endDate, - filters: normalized.filters || [], - }); - } - } + // Pass the chart data (which we already have) and the report config + pushModal('ViewChartUsers', { + chartData: data, + report: { + projectId, + series: reportSeries, + breakdowns: breakdowns || [], + interval, + startDate, + endDate, + range, + previous, + chartType: 'linear', + metric: 'sum', + }, + date: clickedData.date, + }); setClickPosition(null); setClickedData(null); - }, [clickedData, projectId, startDate, endDate, events, series, breakdowns, interval]); + }, [ + clickedData, + projectId, + data, + reportSeries, + breakdowns, + interval, + startDate, + endDate, + range, + previous, + ]); const handleAddReference = useCallback(() => { if (!clickedData) return; diff --git a/apps/start/src/components/report-chart/retention/index.tsx b/apps/start/src/components/report-chart/retention/index.tsx index ff2d777d..58bbffff 100644 --- a/apps/start/src/components/report-chart/retention/index.tsx +++ b/apps/start/src/components/report-chart/retention/index.tsx @@ -12,7 +12,7 @@ import CohortTable from './table'; export function ReportRetentionChart() { const { report: { - events, + series, range, projectId, startDate, @@ -22,8 +22,9 @@ export function ReportRetentionChart() { }, isLazyLoading, } = useReportChartContext(); - const firstEvent = (events[0]?.filters[0]?.value ?? []).map(String); - const secondEvent = (events[1]?.filters[0]?.value ?? []).map(String); + const eventSeries = series.filter((item) => item.type === 'event'); + const firstEvent = (eventSeries[0]?.filters?.[0]?.value ?? []).map(String); + const secondEvent = (eventSeries[1]?.filters?.[0]?.value ?? []).map(String); const isEnabled = firstEvent.length > 0 && secondEvent.length > 0 && !isLazyLoading; const trpc = useTRPC(); diff --git a/apps/start/src/components/report/reportSlice.ts b/apps/start/src/components/report/reportSlice.ts index 93d7436e..b216e549 100644 --- a/apps/start/src/components/report/reportSlice.ts +++ b/apps/start/src/components/report/reportSlice.ts @@ -41,7 +41,7 @@ const initialState: InitialState = { lineType: 'monotone', interval: 'day', breakdowns: [], - events: [], + series: [], range: '30d', startDate: null, endDate: null, @@ -88,10 +88,10 @@ export const reportSlice = createSlice({ state.dirty = true; state.name = action.payload; }, - // Events and Formulas + // Series (Events and Formulas) addEvent: (state, action: PayloadAction>) => { state.dirty = true; - state.events.push({ + state.series.push({ id: shortId(), type: 'event', ...action.payload, @@ -102,7 +102,7 @@ export const reportSlice = createSlice({ action: PayloadAction>, ) => { state.dirty = true; - state.events.push({ + state.series.push({ id: shortId(), ...action.payload, } as IChartEventItem); @@ -113,16 +113,16 @@ export const reportSlice = createSlice({ ) => { state.dirty = true; if (action.payload.type === 'event') { - state.events.push({ - ...action.payload, - filters: action.payload.filters.map((filter) => ({ - ...filter, - id: shortId(), - })), + state.series.push({ + ...action.payload, + filters: action.payload.filters.map((filter) => ({ + ...filter, id: shortId(), + })), + id: shortId(), } as IChartEventItem); } else { - state.events.push({ + state.series.push({ ...action.payload, id: shortId(), } as IChartEventItem); @@ -135,7 +135,7 @@ export const reportSlice = createSlice({ }>, ) => { state.dirty = true; - state.events = state.events.filter( + state.series = state.series.filter( (event) => { // Handle both old format (no type) and new format const eventId = 'type' in event ? event.id : (event as IChartEvent).id; @@ -145,7 +145,7 @@ export const reportSlice = createSlice({ }, changeEvent: (state, action: PayloadAction) => { state.dirty = true; - state.events = state.events.map((event) => { + state.series = state.series.map((event) => { const eventId = 'type' in event ? event.id : (event as IChartEvent).id; if (eventId === action.payload.id) { return action.payload; @@ -293,9 +293,9 @@ export const reportSlice = createSlice({ ) { state.dirty = true; const { fromIndex, toIndex } = action.payload; - const [movedEvent] = state.events.splice(fromIndex, 1); + const [movedEvent] = state.series.splice(fromIndex, 1); if (movedEvent) { - state.events.splice(toIndex, 0, movedEvent); + state.series.splice(toIndex, 0, movedEvent); } }, }, diff --git a/apps/start/src/components/report/sidebar/ReportEvents.tsx b/apps/start/src/components/report/sidebar/ReportEvents.tsx index 50c9d1d3..20004f7e 100644 --- a/apps/start/src/components/report/sidebar/ReportEvents.tsx +++ b/apps/start/src/components/report/sidebar/ReportEvents.tsx @@ -266,60 +266,60 @@ export function ReportEvents() { ) : ( <> - { - dispatch( - changeEvent( - Array.isArray(value) - ? { + } + onChange={(value) => { + dispatch( + changeEvent( + Array.isArray(value) + ? { id: normalized.id, type: 'event', - segment: 'user', - filters: [ - { - name: 'name', - operator: 'is', - value: value, - }, - ], - name: '*', - } - : { + segment: 'user', + filters: [ + { + name: 'name', + operator: 'is', + value: value, + }, + ], + name: '*', + } + : { ...normalized, type: 'event', - name: value, - filters: [], - }, - ), - ); - }} - items={eventNames} - placeholder="Select event" - /> - {showDisplayNameInput && ( - + {showDisplayNameInput && ( + { - dispatchChangeEvent({ + onChange={(e) => { + dispatchChangeEvent({ ...(normalized as IChartEventItem & { type: 'event' }), - displayName: e.target.value, - }); - }} - /> - )} + displayName: e.target.value, + }); + }} + /> + )} )} @@ -328,38 +328,38 @@ export function ReportEvents() { })}
- { - if (isSelectManyEvents) { - dispatch( - addEvent({ - segment: 'user', - name: value, - filters: [ - { - name: 'name', - operator: 'is', - value: [value], - }, - ], - }), - ); - } else { - dispatch( - addEvent({ - name: value, - segment: 'event', - filters: [], - }), - ); - } - }} - placeholder="Select event" - items={eventNames} - /> + { + if (isSelectManyEvents) { + dispatch( + addEvent({ + segment: 'user', + name: value, + filters: [ + { + name: 'name', + operator: 'is', + value: [value], + }, + ], + }), + ); + } else { + dispatch( + addEvent({ + name: value, + segment: 'event', + filters: [], + }), + ); + } + }} + placeholder="Select event" + items={eventNames} + /> {showFormula && ( + {props.children} +
+ + {/* Segment and Filter buttons - only for events */} + {chartEvent && (showSegment || showAddFilter) && ( +
+ {showSegment && ( + { + dispatch( + changeEvent({ + ...chartEvent, + segment, + }), + ); + }} + /> + )} + {showAddFilter && ( + { + dispatch( + changeEvent({ + ...chartEvent, + filters: [ + ...chartEvent.filters, + { + id: shortId(), + name: action.value, + operator: 'is', + value: [], + }, + ], + }), + ); + }} + > + {(setOpen) => ( + + )} + + )} + + {showSegment && chartEvent.segment.startsWith('property_') && ( + + )} +
+ )} + + {/* Filters - only for events */} + {chartEvent && !isSelectManyEvents && } +
+ ); +} + +export function ReportSeries() { + const selectedSeries = useSelector((state) => state.report.series); + const chartType = useSelector((state) => state.report.chartType); + const dispatch = useDispatch(); + const { projectId } = useAppParams(); + const eventNames = useEventNames({ + projectId, + }); + + const showSegment = !['retention', 'funnel'].includes(chartType); + const showAddFilter = !['retention'].includes(chartType); + const showDisplayNameInput = !['retention'].includes(chartType); + const isAddEventDisabled = + (chartType === 'retention' || chartType === 'conversion') && + selectedSeries.length >= 2; + const dispatchChangeEvent = useDebounceFn((event: IChartEvent) => { + dispatch(changeEvent(event)); + }); + const isSelectManyEvents = chartType === 'retention'; + + const sensors = useSensors( + useSensor(PointerSensor), + useSensor(KeyboardSensor, { + coordinateGetter: sortableKeyboardCoordinates, + }), + ); + + const handleDragEnd = (event: DragEndEvent) => { + const { active, over } = event; + + if (over && active.id !== over.id) { + const oldIndex = selectedSeries.findIndex((e) => e.id === active.id); + const newIndex = selectedSeries.findIndex((e) => e.id === over.id); + + dispatch(reorderEvents({ fromIndex: oldIndex, toIndex: newIndex })); + } + }; + + const handleMore = (event: IChartEventItem | IChartEvent) => { + const callback: ReportEventMoreProps['onClick'] = (action) => { + switch (action) { + case 'remove': { + return dispatch( + removeEvent({ + id: 'type' in event ? event.id : (event as IChartEvent).id, + }), + ); + } + case 'duplicate': { + const normalized = + 'type' in event ? event : { ...event, type: 'event' as const }; + return dispatch(duplicateEvent(normalized)); + } + } + }; + + return callback; + }; + + const dispatchChangeFormula = useDebounceFn((formula: IChartFormula) => { + dispatch(changeEvent(formula)); + }); + + const showFormula = + chartType !== 'conversion' && + chartType !== 'funnel' && + chartType !== 'retention'; + + return ( +
+

Metrics

+ + ({ + id: ('type' in e ? e.id : (e as IChartEvent).id) ?? '', + }))} + strategy={verticalListSortingStrategy} + > +
+ {selectedSeries.map((event, index) => { + const isFormula = event.type === 'formula'; + + return ( + + {isFormula ? ( + <> +
+ { + dispatchChangeFormula({ + ...event, + formula: value, + }); + }} + /> + {showDisplayNameInput && ( + { + dispatchChangeFormula({ + ...event, + displayName: e.target.value, + }); + }} + /> + )} +
+ + + ) : ( + <> + { + dispatch( + changeEvent( + Array.isArray(value) + ? { + id: event.id, + type: 'event', + segment: 'user', + filters: [ + { + name: 'name', + operator: 'is', + value: value, + }, + ], + name: '*', + } + : { + ...event, + type: 'event', + name: value, + filters: [], + }, + ), + ); + }} + items={eventNames} + placeholder="Select event" + /> + {showDisplayNameInput && ( + { + dispatchChangeEvent({ + ...(event as IChartEventItem & { + type: 'event'; + }), + displayName: e.target.value, + }); + }} + /> + )} + + + )} + + ); + })} + +
+ { + if (isSelectManyEvents) { + dispatch( + addEvent({ + segment: 'user', + name: value, + filters: [ + { + name: 'name', + operator: 'is', + value: [value], + }, + ], + }), + ); + } else { + dispatch( + addEvent({ + name: value, + segment: 'event', + filters: [], + }), + ); + } + }} + placeholder="Select event" + items={eventNames} + /> + {showFormula && ( + + )} +
+
+ + +
+ ); +} diff --git a/apps/start/src/components/report/sidebar/ReportSidebar.tsx b/apps/start/src/components/report/sidebar/ReportSidebar.tsx index 5d03261c..2d99fc8b 100644 --- a/apps/start/src/components/report/sidebar/ReportSidebar.tsx +++ b/apps/start/src/components/report/sidebar/ReportSidebar.tsx @@ -3,7 +3,7 @@ import { SheetClose, SheetFooter } from '@/components/ui/sheet'; import { useSelector } from '@/redux'; import { ReportBreakdowns } from './ReportBreakdowns'; -import { ReportEvents } from './ReportEvents'; +import { ReportSeries } from './ReportSeries'; import { ReportFormula } from './ReportFormula'; import { ReportSettings } from './ReportSettings'; @@ -13,7 +13,7 @@ export function ReportSidebar() { return ( <>
- + {showBreakdown && }
diff --git a/apps/start/src/components/ui/input-enter.tsx b/apps/start/src/components/ui/input-enter.tsx index ad2617ee..ec2c6f33 100644 --- a/apps/start/src/components/ui/input-enter.tsx +++ b/apps/start/src/components/ui/input-enter.tsx @@ -4,7 +4,7 @@ import { AnimatePresence } from 'framer-motion'; import { RefreshCcwIcon } from 'lucide-react'; import { type InputHTMLAttributes, useEffect, useState } from 'react'; import { Badge } from './badge'; -import { Input } from './input'; +import { Input, type InputProps } from './input'; export function InputEnter({ value, @@ -13,7 +13,7 @@ export function InputEnter({ }: { value: string | undefined; onChangeValue: (value: string) => void; -} & InputHTMLAttributes) { +} & InputProps) { const [internalValue, setInternalValue] = useState(value ?? ''); useEffect(() => { @@ -33,7 +33,6 @@ export function InputEnter({ onChangeValue(internalValue); } }} - size="default" />
diff --git a/apps/start/src/modals/view-chart-users.tsx b/apps/start/src/modals/view-chart-users.tsx index 1a65897e..76148046 100644 --- a/apps/start/src/modals/view-chart-users.tsx +++ b/apps/start/src/modals/view-chart-users.tsx @@ -1,62 +1,197 @@ import { ButtonContainer } from '@/components/button-container'; import { Button } from '@/components/ui/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; import { useTRPC } from '@/integrations/trpc/react'; +import type { IChartData } from '@/trpc/client'; +import type { IChartEvent, IChartInput } from '@openpanel/validation'; import { useQuery } from '@tanstack/react-query'; import { UsersIcon } from 'lucide-react'; +import { useMemo, useState } from 'react'; import { popModal } from '.'; import { ModalContent, ModalHeader } from './Modal/Container'; -import type { IChartEvent } from '@openpanel/validation'; interface ViewChartUsersProps { - projectId: string; - event: IChartEvent; + chartData: IChartData; + report: IChartInput; date: string; - breakdowns?: Array<{ id?: string; name: string }>; - interval: string; - startDate: string; - endDate: string; - filters?: Array<{ - id?: string; - name: string; - operator: string; - value: Array; - }>; } export default function ViewChartUsers({ - projectId, - event, + chartData, + report, date, - breakdowns = [], - interval, - startDate, - endDate, - filters = [], }: ViewChartUsersProps) { const trpc = useTRPC(); - const query = useQuery( - trpc.chart.getProfiles.queryOptions({ - projectId, - event, - date, - breakdowns, - interval: interval as any, - startDate, - endDate, - filters, - }), + + // Group series by base event/formula (ignoring breakdowns) + const baseSeries = useMemo(() => { + const grouped = new Map< + string, + { + baseName: string; + baseEventId: string; + reportSerie: IChartInput['series'][0] | undefined; + breakdownSeries: Array<{ + serie: IChartData['series'][0]; + breakdowns: Record | undefined; + }>; + } + >(); + + chartData.series.forEach((serie) => { + const baseEventId = serie.event.id || ''; + const baseName = serie.names[0] || 'Unnamed Serie'; + + if (!grouped.has(baseEventId)) { + const reportSerie = report.series.find((ss) => ss.id === baseEventId); + grouped.set(baseEventId, { + baseName, + baseEventId, + reportSerie, + breakdownSeries: [], + }); + } + + const group = grouped.get(baseEventId); + if (!group) return; + // Extract breakdowns from serie.event.breakdowns (set in format.ts) + const breakdowns = (serie.event as any).breakdowns; + + group.breakdownSeries.push({ + serie, + breakdowns, + }); + }); + + return Array.from(grouped.values()); + }, [chartData.series, report.series, report.breakdowns]); + + const [selectedBaseSerieId, setSelectedBaseSerieId] = useState( + null, + ); + const [selectedBreakdownIndex, setSelectedBreakdownIndex] = useState< + number | null + >(null); + + const selectedBaseSerie = useMemo( + () => baseSeries.find((bs) => bs.baseEventId === selectedBaseSerieId), + [baseSeries, selectedBaseSerieId], ); - const profiles = query.data ?? []; + const selectedBreakdown = useMemo(() => { + if ( + !selectedBaseSerie || + selectedBreakdownIndex === null || + !selectedBaseSerie.breakdownSeries[selectedBreakdownIndex] + ) { + return null; + } + return selectedBaseSerie.breakdownSeries[selectedBreakdownIndex]; + }, [selectedBaseSerie, selectedBreakdownIndex]); + + // Reset breakdown selection when base serie changes + const handleBaseSerieChange = (value: string) => { + setSelectedBaseSerieId(value); + setSelectedBreakdownIndex(null); + }; + + const selectedSerie = selectedBreakdown || selectedBaseSerie; + + const profilesQuery = useQuery( + trpc.chart.getProfiles.queryOptions( + { + projectId: report.projectId, + date: date, + series: + selectedSerie && + selectedBaseSerie?.reportSerie && + selectedBaseSerie.reportSerie.type === 'event' + ? [selectedBaseSerie.reportSerie] + : [], + breakdowns: selectedBreakdown?.breakdowns, + interval: report.interval, + }, + { + enabled: + !!selectedSerie && + !!selectedBaseSerie?.reportSerie && + selectedBaseSerie.reportSerie.type === 'event', + }, + ), + ); + + const profiles = profilesQuery.data ?? []; return ( - + +

+ Users who performed actions on {new Date(date).toLocaleDateString()} +

- {query.isLoading ? ( + {baseSeries.length > 0 && ( +
+
+ + +
+ + {selectedBaseSerie && + selectedBaseSerie.breakdownSeries.length > 1 && ( +
+ + +
+ )} +
+ )} + {profilesQuery.isLoading ? (
Loading users...
@@ -109,4 +244,3 @@ export default function ViewChartUsers({ ); } - diff --git a/packages/db/index.ts b/packages/db/index.ts index c0c91330..58042d3f 100644 --- a/packages/db/index.ts +++ b/packages/db/index.ts @@ -3,6 +3,7 @@ export * from './src/clickhouse/client'; export * from './src/clickhouse/csv'; export * from './src/sql-builder'; export * from './src/services/chart.service'; +export * from './src/engine'; export * from './src/services/clients.service'; export * from './src/services/dashboard.service'; export * from './src/services/event.service'; diff --git a/packages/db/package.json b/packages/db/package.json index 284b6d96..fb4cd393 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -25,6 +25,7 @@ "@prisma/extension-read-replicas": "^0.4.1", "fast-deep-equal": "^3.1.3", "jiti": "^2.4.1", + "mathjs": "^12.3.2", "prisma-json-types-generator": "^3.1.1", "ramda": "^0.29.1", "sqlstring": "^2.3.3", diff --git a/packages/db/src/clickhouse/query-builder.ts b/packages/db/src/clickhouse/query-builder.ts index 351ef6b1..ba2416c7 100644 --- a/packages/db/src/clickhouse/query-builder.ts +++ b/packages/db/src/clickhouse/query-builder.ts @@ -731,6 +731,7 @@ clix.toInterval = (node: string, interval: IInterval) => { }; clix.toDate = (node: string, interval: IInterval) => { switch (interval) { + case 'day': case 'week': case 'month': { return `toDate(${node})`; diff --git a/packages/db/src/engine/compute.ts b/packages/db/src/engine/compute.ts new file mode 100644 index 00000000..ed50e323 --- /dev/null +++ b/packages/db/src/engine/compute.ts @@ -0,0 +1,165 @@ +import { round } from '@openpanel/common'; +import { alphabetIds } from '@openpanel/constants'; +import type { IChartFormula } from '@openpanel/validation'; +import * as mathjs from 'mathjs'; +import type { ConcreteSeries } from './types'; + +/** + * Compute formula series from fetched event series + * Formulas reference event series using alphabet IDs (A, B, C, etc.) + */ +export function compute( + fetchedSeries: ConcreteSeries[], + definitions: Array<{ + type: 'event' | 'formula'; + id?: string; + formula?: string; + }>, +): ConcreteSeries[] { + const results: ConcreteSeries[] = [...fetchedSeries]; + + // Process formulas in order (they can reference previous formulas) + definitions.forEach((definition, formulaIndex) => { + if (definition.type !== 'formula') { + return; + } + + const formula = definition as IChartFormula; + if (!formula.formula) { + return; + } + + // Group ALL series (events + previously computed formulas) by breakdown signature + // Series with the same breakdown values should be computed together + const seriesByBreakdown = new Map(); + + // Include both fetched event series AND previously computed formulas + const allSeries = [ + ...fetchedSeries, + ...results.filter((s) => s.definitionIndex < formulaIndex), + ]; + + allSeries.forEach((serie) => { + // Create breakdown signature: skip first name part (event/formula name) and use breakdown values + // If name.length === 1, it means no breakdowns (just event name) + // If name.length > 1, name[0] is event name, name[1+] are breakdown values + const breakdownSignature = + serie.name.length > 1 ? serie.name.slice(1).join(':::') : ''; + + if (!seriesByBreakdown.has(breakdownSignature)) { + seriesByBreakdown.set(breakdownSignature, []); + } + seriesByBreakdown.get(breakdownSignature)!.push(serie); + }); + + // Compute formula for each breakdown group + for (const [breakdownSignature, breakdownSeries] of seriesByBreakdown) { + // Map series by their definition index for formula evaluation + const seriesByIndex = new Map(); + breakdownSeries.forEach((serie) => { + seriesByIndex.set(serie.definitionIndex, serie); + }); + + // Get all unique dates across all series in this breakdown group + const allDates = new Set(); + breakdownSeries.forEach((serie) => { + serie.data.forEach((item) => { + allDates.add(item.date); + }); + }); + + const sortedDates = Array.from(allDates).sort( + (a, b) => new Date(a).getTime() - new Date(b).getTime(), + ); + + // Calculate formula for each date + const formulaData = sortedDates.map((date) => { + const scope: Record = {}; + + // Build scope using alphabet IDs (A, B, C, etc.) + definitions.slice(0, formulaIndex).forEach((depDef, depIndex) => { + const readableId = alphabetIds[depIndex]; + if (!readableId) { + return; + } + + // Find the series for this dependency in the current breakdown group + const depSeries = seriesByIndex.get(depIndex); + if (depSeries) { + const dataPoint = depSeries.data.find((d) => d.date === date); + scope[readableId] = dataPoint?.count ?? 0; + } else { + // Could be a formula from a previous breakdown group - find it in results + // Match by definitionIndex AND breakdown signature + const formulaSerie = results.find( + (s) => + s.definitionIndex === depIndex && + 'type' in s.definition && + s.definition.type === 'formula' && + s.name.slice(1).join(':::') === breakdownSignature, + ); + if (formulaSerie) { + const dataPoint = formulaSerie.data.find((d) => d.date === date); + scope[readableId] = dataPoint?.count ?? 0; + } else { + scope[readableId] = 0; + } + } + }); + + // Evaluate formula + let count: number; + try { + count = mathjs + .parse(formula.formula) + .compile() + .evaluate(scope) as number; + } catch (error) { + count = 0; + } + + return { + date, + count: + Number.isNaN(count) || !Number.isFinite(count) + ? 0 + : round(count, 2), + total_count: breakdownSeries[0]?.data.find((d) => d.date === date) + ?.total_count, + }; + }); + + // Create concrete series for this formula + const templateSerie = breakdownSeries[0]!; + + // Extract breakdown values from template series name + // name[0] is event/formula name, name[1+] are breakdown values + const breakdownValues = + templateSerie.name.length > 1 ? templateSerie.name.slice(1) : []; + + const formulaName = + breakdownValues.length > 0 + ? [formula.displayName || formula.formula, ...breakdownValues] + : [formula.displayName || formula.formula]; + + const formulaSeries: ConcreteSeries = { + id: `formula-${formula.id ?? formulaIndex}-${breakdownSignature || 'default'}`, + definitionId: + formula.id ?? alphabetIds[formulaIndex] ?? `formula-${formulaIndex}`, + definitionIndex: formulaIndex, + name: formulaName, + context: { + filters: templateSerie.context.filters, + breakdownValue: templateSerie.context.breakdownValue, + breakdowns: templateSerie.context.breakdowns, + }, + data: formulaData, + definition: formula, + }; + + results.push(formulaSeries); + } + }); + + return results; +} diff --git a/packages/db/src/engine/fetch.ts b/packages/db/src/engine/fetch.ts new file mode 100644 index 00000000..954c98e9 --- /dev/null +++ b/packages/db/src/engine/fetch.ts @@ -0,0 +1,151 @@ +import type { ISerieDataItem } from '@openpanel/common'; +import { groupByLabels } from '@openpanel/common'; +import { alphabetIds } from '@openpanel/constants'; +import type { IGetChartDataInput } from '@openpanel/validation'; +import { chQuery } from '../clickhouse/client'; +import { getChartSql } from '../services/chart.service'; +import type { ConcreteSeries, Plan } from './types'; + +/** + * Fetch data for all event series in the plan + * This handles breakdown expansion automatically via groupByLabels + */ +export async function fetch(plan: Plan): Promise { + const results: ConcreteSeries[] = []; + + // Process each event definition + for (let i = 0; i < plan.definitions.length; i++) { + const definition = plan.definitions[i]!; + + if (definition.type !== 'event') { + // Skip formulas - they'll be handled in compute stage + continue; + } + + const event = definition as typeof definition & { type: 'event' }; + + // Find the corresponding concrete series placeholder + const placeholder = plan.concreteSeries.find( + (cs) => cs.definitionId === definition.id, + ); + + if (!placeholder) { + continue; + } + + // Build query input + const queryInput: IGetChartDataInput = { + event: { + id: event.id, + name: event.name, + segment: event.segment, + filters: event.filters, + displayName: event.displayName, + property: event.property, + }, + projectId: plan.input.projectId, + startDate: plan.input.startDate, + endDate: plan.input.endDate, + breakdowns: plan.input.breakdowns, + interval: plan.input.interval, + chartType: plan.input.chartType, + metric: plan.input.metric, + previous: plan.input.previous ?? false, + limit: plan.input.limit, + offset: plan.input.offset, + criteria: plan.input.criteria, + funnelGroup: plan.input.funnelGroup, + funnelWindow: plan.input.funnelWindow, + }; + + // Execute query + let queryResult = await chQuery( + getChartSql({ ...queryInput, timezone: plan.timezone }), + { + session_timezone: plan.timezone, + }, + ); + + // Fallback: if no results with breakdowns, try without breakdowns + if (queryResult.length === 0 && plan.input.breakdowns.length > 0) { + queryResult = await chQuery( + getChartSql({ + ...queryInput, + breakdowns: [], + timezone: plan.timezone, + }), + { + session_timezone: plan.timezone, + }, + ); + } + + // Group by labels (handles breakdown expansion) + const groupedSeries = groupByLabels(queryResult); + + // Create concrete series for each grouped result + groupedSeries.forEach((grouped) => { + // Extract breakdown value from name array + // If breakdowns exist, name[0] is event name, name[1+] are breakdown values + const breakdownValue = + plan.input.breakdowns.length > 0 && grouped.name.length > 1 + ? grouped.name.slice(1).join(' - ') + : undefined; + + // Build breakdowns object: { country: 'SE', path: '/ewoqmepwq' } + const breakdowns: Record | undefined = + plan.input.breakdowns.length > 0 && grouped.name.length > 1 + ? {} + : undefined; + + if (breakdowns) { + plan.input.breakdowns.forEach((breakdown, idx) => { + const breakdownNamePart = grouped.name[idx + 1]; + if (breakdownNamePart) { + breakdowns[breakdown.name] = breakdownNamePart; + } + }); + } + + // Build filters including breakdown value + const filters = [...event.filters]; + if (breakdownValue && plan.input.breakdowns.length > 0) { + // Add breakdown filter + plan.input.breakdowns.forEach((breakdown, idx) => { + const breakdownNamePart = grouped.name[idx + 1]; + if (breakdownNamePart) { + filters.push({ + id: `breakdown-${idx}`, + name: breakdown.name, + operator: 'is', + value: [breakdownNamePart], + }); + } + }); + } + + const concrete: ConcreteSeries = { + id: `${placeholder.id}-${grouped.name.join('-')}`, + definitionId: definition.id ?? alphabetIds[i] ?? `series-${i}`, + definitionIndex: i, + name: grouped.name, + context: { + event: event.name, + filters, + breakdownValue, + breakdowns, + }, + data: grouped.data.map((item) => ({ + date: item.date, + count: item.count, + total_count: item.total_count, + })), + definition, + }; + + results.push(concrete); + }); + } + + return results; +} diff --git a/packages/db/src/engine/format.ts b/packages/db/src/engine/format.ts new file mode 100644 index 00000000..ff452747 --- /dev/null +++ b/packages/db/src/engine/format.ts @@ -0,0 +1,141 @@ +import { + average, + getPreviousMetric, + max, + min, + round, + slug, + sum, +} from '@openpanel/common'; +import { alphabetIds } from '@openpanel/constants'; +import type { FinalChart } from '@openpanel/validation'; +import type { ConcreteSeries } from './types'; + +/** + * Format concrete series into FinalChart format (backward compatible) + * TODO: Migrate frontend to use cleaner ChartResponse format + */ +export function format( + concreteSeries: ConcreteSeries[], + definitions: Array<{ + id?: string; + type: 'event' | 'formula'; + displayName?: string; + formula?: string; + name?: string; + }>, + includeAlphaIds: boolean, + previousSeries: ConcreteSeries[] | null = null, +): FinalChart { + const series = concreteSeries.map((cs) => { + // Find definition for this series + const definition = definitions[cs.definitionIndex]; + const alphaId = includeAlphaIds + ? alphabetIds[cs.definitionIndex] + : undefined; + + // Build display name with optional alpha ID + let displayName: string[]; + + // Replace the first name (which is the event name) with the display name if it exists + const names = cs.name.slice(0); + if (cs.definition.displayName) { + names.splice(0, 1, cs.definition.displayName); + } + // Add the alpha ID to the first name if it exists + if (alphaId) { + displayName = [`(${alphaId}) ${names[0]}`, ...names.slice(1)]; + } else { + displayName = names; + } + + // Calculate metrics for this series + const counts = cs.data.map((d) => d.count); + const metrics = { + sum: sum(counts), + average: round(average(counts), 2), + min: min(counts), + max: max(counts), + count: cs.data.find((item) => !!item.total_count)?.total_count, + }; + + // Build event object for compatibility + const eventName = + definition?.type === 'formula' + ? definition.displayName || definition.formula || 'Formula' + : definition?.name || cs.context.event || 'unknown'; + + // Find matching previous series + const previousSerie = previousSeries?.find( + (ps) => + ps.definitionIndex === cs.definitionIndex && + ps.name.slice(1).join(':::') === cs.name.slice(1).join(':::'), + ); + + return { + id: cs.id, + names: displayName, + // TODO: Do we need this now? + event: { + id: definition?.id, + name: eventName, + breakdowns: cs.context.breakdowns, + }, + metrics: { + ...metrics, + ...(previousSerie + ? { + previous: { + sum: getPreviousMetric( + metrics.sum, + sum(previousSerie.data.map((d) => d.count)), + ), + average: getPreviousMetric( + metrics.average, + round(average(previousSerie.data.map((d) => d.count)), 2), + ), + min: getPreviousMetric( + metrics.min, + min(previousSerie.data.map((d) => d.count)), + ), + max: getPreviousMetric( + metrics.max, + max(previousSerie.data.map((d) => d.count)), + ), + count: getPreviousMetric( + metrics.count ?? 0, + previousSerie.data.find((item) => !!item.total_count) + ?.total_count ?? null, + ), + }, + } + : {}), + }, + data: cs.data.map((item, index) => ({ + date: item.date, + count: item.count, + previous: previousSerie?.data[index] + ? getPreviousMetric( + item.count, + previousSerie.data[index]?.count ?? null, + ) + : undefined, + })), + }; + }); + + // Calculate global metrics + const allValues = concreteSeries.flatMap((cs) => cs.data.map((d) => d.count)); + const globalMetrics = { + sum: sum(allValues), + average: round(average(allValues), 2), + min: min(allValues), + max: max(allValues), + count: undefined as number | undefined, + }; + + return { + series, + metrics: globalMetrics, + }; +} diff --git a/packages/db/src/engine/index.ts b/packages/db/src/engine/index.ts new file mode 100644 index 00000000..04461484 --- /dev/null +++ b/packages/db/src/engine/index.ts @@ -0,0 +1,77 @@ +import { getPreviousMetric } from '@openpanel/common'; + +import type { FinalChart, IChartInput } from '@openpanel/validation'; +import { getChartPrevStartEndDate } from '../services/chart.service'; +import { + getOrganizationSubscriptionChartEndDate, + getSettingsForProject, +} from '../services/organization.service'; +import { compute } from './compute'; +import { fetch } from './fetch'; +import { format } from './format'; +import { normalize } from './normalize'; +import { plan } from './plan'; +import type { ConcreteSeries } from './types'; + +/** + * Chart Engine - Main entry point + * Executes the pipeline: normalize -> plan -> fetch -> compute -> format + */ +export async function executeChart(input: IChartInput): Promise { + const { timezone } = await getSettingsForProject(input.projectId); + + // Stage 1: Normalize input + const normalized = await normalize(input); + + // Handle subscription end date limit + const endDate = await getOrganizationSubscriptionChartEndDate( + input.projectId, + normalized.endDate, + ); + if (endDate) { + normalized.endDate = endDate; + } + + // Stage 2: Create execution plan + const executionPlan = await plan(normalized); + + // Stage 3: Fetch data for event series (current period) + const fetchedSeries = await fetch(executionPlan); + + // Stage 4: Compute formula series + const computedSeries = compute(fetchedSeries, executionPlan.definitions); + + // Stage 5: Fetch previous period if requested + let previousSeries: ConcreteSeries[] | null = null; + if (input.previous) { + const currentPeriod = { + startDate: normalized.startDate, + endDate: normalized.endDate, + }; + const previousPeriod = getChartPrevStartEndDate(currentPeriod); + + const previousPlan = await plan({ + ...normalized, + ...previousPeriod, + }); + + const previousFetched = await fetch(previousPlan); + previousSeries = compute(previousFetched, previousPlan.definitions); + } + + // Stage 6: Format final output with previous period data + const includeAlphaIds = executionPlan.definitions.length > 1; + const response = format( + computedSeries, + executionPlan.definitions, + includeAlphaIds, + previousSeries, + ); + + return response; +} + +// Export as ChartEngine for backward compatibility +export const ChartEngine = { + execute: executeChart, +}; diff --git a/packages/db/src/engine/normalize.ts b/packages/db/src/engine/normalize.ts new file mode 100644 index 00000000..ed37a3e6 --- /dev/null +++ b/packages/db/src/engine/normalize.ts @@ -0,0 +1,66 @@ +import { alphabetIds } from '@openpanel/constants'; +import type { + IChartEvent, + IChartEventItem, + IChartInput, + IChartInputWithDates, +} from '@openpanel/validation'; +import { getChartStartEndDate } from '../services/chart.service'; +import { getSettingsForProject } from '../services/organization.service'; +import type { SeriesDefinition } from './types'; + +export type NormalizedInput = Awaited>; + +/** + * Normalize a chart input into a clean structure with dates and normalized series + */ +export async function normalize( + input: IChartInput, +): Promise { + const { timezone } = await getSettingsForProject(input.projectId); + const { startDate, endDate } = getChartStartEndDate( + { + range: input.range, + startDate: input.startDate ?? undefined, + endDate: input.endDate ?? undefined, + }, + timezone, + ); + + // Get series from input (handles both 'series' and 'events' fields) + // The schema preprocessing should have already converted 'events' to 'series', but handle both for safety + const rawSeries = (input as any).series ?? (input as any).events ?? []; + + // Normalize each series item + const normalizedSeries: SeriesDefinition[] = rawSeries.map( + (item: any, index: number) => { + // If item already has type field, it's the new format + if (item && typeof item === 'object' && 'type' in item) { + return { + ...item, + id: item.id ?? alphabetIds[index] ?? `series-${index}`, + } as SeriesDefinition; + } + + // Old format without type field - assume it's an event + const event = item as Partial; + return { + type: 'event', + id: event.id ?? alphabetIds[index] ?? `series-${index}`, + name: event.name || 'unknown_event', + segment: event.segment ?? 'event', + filters: event.filters ?? [], + displayName: event.displayName, + property: event.property, + } as SeriesDefinition; + }, + ); + + return { + ...input, + series: normalizedSeries, + startDate, + endDate, + }; +} + diff --git a/packages/db/src/engine/plan.ts b/packages/db/src/engine/plan.ts new file mode 100644 index 00000000..d3581fbf --- /dev/null +++ b/packages/db/src/engine/plan.ts @@ -0,0 +1,59 @@ +import { slug } from '@openpanel/common'; +import { alphabetIds } from '@openpanel/constants'; +import type { + IChartBreakdown, + IChartEvent, + IChartEventItem, +} from '@openpanel/validation'; +import { getSettingsForProject } from '../services/organization.service'; +import type { ConcreteSeries, Plan } from './types'; +import type { NormalizedInput } from './normalize'; + +/** + * Create an execution plan from normalized input + * This sets up ConcreteSeries placeholders - actual breakdown expansion happens during fetch + */ +export async function plan( + normalized: NormalizedInput, +): Promise { + const { timezone } = await getSettingsForProject(normalized.projectId); + + const concreteSeries: ConcreteSeries[] = []; + + // Create concrete series placeholders for each definition + normalized.series.forEach((definition, index) => { + if (definition.type === 'event') { + const event = definition as IChartEventItem & { type: 'event' }; + + // For events, create a placeholder + // If breakdowns exist, fetch will return multiple series (one per breakdown value) + // If no breakdowns, fetch will return one series + const concrete: ConcreteSeries = { + id: `${slug(event.name)}-${event.id ?? index}`, + definitionId: event.id ?? alphabetIds[index] ?? `series-${index}`, + definitionIndex: index, + name: [event.displayName || event.name], + context: { + event: event.name, + filters: [...event.filters], + }, + data: [], // Will be populated by fetch stage + definition, + }; + concreteSeries.push(concrete); + } else { + // For formulas, we'll create placeholders during compute stage + // Formulas depend on event series, so we skip them here + } + }); + + return { + concreteSeries, + definitions: normalized.series, + input: normalized, + timezone, + }; +} + +export type NormalizedInput = Awaited>; + diff --git a/packages/db/src/engine/types.ts b/packages/db/src/engine/types.ts new file mode 100644 index 00000000..7aad9c31 --- /dev/null +++ b/packages/db/src/engine/types.ts @@ -0,0 +1,85 @@ +import type { + IChartBreakdown, + IChartEvent, + IChartEventFilter, + IChartEventItem, + IChartFormula, + IChartInput, + IChartInputWithDates, +} from '@openpanel/validation'; + +/** + * Series Definition - The input representation of what the user wants + * This is what comes from the frontend (events or formulas) + */ +export type SeriesDefinition = IChartEventItem; + +/** + * Concrete Series - A resolved series that will be displayed as a line/bar on the chart + * When breakdowns exist, one SeriesDefinition can expand into multiple ConcreteSeries + */ +export type ConcreteSeries = { + id: string; + definitionId: string; // ID of the SeriesDefinition this came from + definitionIndex: number; // Index in the original series array (for A, B, C references) + name: string[]; // Display name parts: ["Session Start", "Chrome"] or ["Formula 1"] + + // Context for Drill-down / Profiles + // This contains everything needed to query 'who are these users?' + context: { + event?: string; // Event name (if this is an event series) + filters: IChartEventFilter[]; // All filters including breakdown value + breakdownValue?: string; // The breakdown value for this concrete series (deprecated, use breakdowns instead) + breakdowns?: Record; // Breakdown keys and values: { country: 'SE', path: '/ewoqmepwq' } + }; + + // Data points for this series + data: Array<{ + date: string; + count: number; + total_count?: number; + }>; + + // The original definition (event or formula) + definition: SeriesDefinition; +}; + +/** + * Plan - The execution plan after normalization and expansion + */ +export type Plan = { + concreteSeries: ConcreteSeries[]; + definitions: SeriesDefinition[]; + input: IChartInputWithDates; + timezone: string; +}; + +/** + * Chart Response - The final output format + */ +export type ChartResponse = { + series: Array<{ + id: string; + name: string[]; + data: Array<{ + date: string; + value: number; + previous?: number; + }>; + summary: { + total: number; + average: number; + min: number; + max: number; + count?: number; + }; + context?: ConcreteSeries['context']; // Include context for drill-down + }>; + summary: { + total: number; + average: number; + min: number; + max: number; + }; +}; + diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts index fddc1577..5f70b65b 100644 --- a/packages/db/src/services/chart.service.ts +++ b/packages/db/src/services/chart.service.ts @@ -155,7 +155,8 @@ export function getChartSql({ } breakdowns.forEach((breakdown, index) => { - const key = `label_${index}`; + // Breakdowns start at label_1 (label_0 is reserved for event name) + const key = `label_${index + 1}`; sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`; sb.groupBy[key] = `${key}`; }); diff --git a/packages/db/src/services/conversion.service.ts b/packages/db/src/services/conversion.service.ts index 4046ced5..6df418ca 100644 --- a/packages/db/src/services/conversion.service.ts +++ b/packages/db/src/services/conversion.service.ts @@ -1,5 +1,5 @@ import { NOT_SET_VALUE } from '@openpanel/constants'; -import type { IChartInput } from '@openpanel/validation'; +import type { IChartEvent, IChartInput } from '@openpanel/validation'; import { omit } from 'ramda'; import { TABLE_NAMES, ch } from '../clickhouse/client'; import { clix } from '../clickhouse/query-builder'; @@ -17,7 +17,8 @@ export class ConversionService { endDate, funnelGroup, funnelWindow = 24, - events, + series, + events, // Backward compatibility - use series if available breakdowns = [], interval, timezone, @@ -30,7 +31,12 @@ export class ConversionService { ); const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`); - if (events.length !== 2) { + // Use series if available, otherwise fall back to events (backward compat) + const eventSeries = (series ?? events ?? []).filter( + (item): item is IChartEvent => item.type === 'event', + ) as IChartEvent[]; + + if (eventSeries.length !== 2) { throw new Error('events must be an array of two events'); } @@ -38,8 +44,8 @@ export class ConversionService { throw new Error('startDate and endDate are required'); } - const eventA = events[0]!; - const eventB = events[1]!; + const eventA = eventSeries[0]!; + const eventB = eventSeries[1]!; const whereA = Object.values( getEventFiltersWhereClause(eventA.filters), ).join(' AND '); diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.ts index f2841f38..c7ed74ac 100644 --- a/packages/db/src/services/funnel.service.ts +++ b/packages/db/src/services/funnel.service.ts @@ -20,7 +20,7 @@ export class FunnelService { : ['session_id', 'session_id']; } - private getFunnelConditions(events: IChartEvent[]) { + private getFunnelConditions(events: IChartEvent[] = []) { return events.map((event) => { const { sb, getWhere } = createSqlBuilder(); sb.where = getEventFiltersWhereClause(event.filters); @@ -110,7 +110,8 @@ export class FunnelService { projectId, startDate, endDate, - events, + series, + events, // Backward compatibility - use series if available funnelWindow = 24, funnelGroup, breakdowns = [], @@ -120,15 +121,20 @@ export class FunnelService { throw new Error('startDate and endDate are required'); } - if (events.length === 0) { + // Use series if available, otherwise fall back to events (backward compat) + const eventSeries = (series ?? events ?? []).filter( + (item): item is IChartEvent => item.type === 'event', + ) as IChartEvent[]; + + if (eventSeries.length === 0) { throw new Error('events are required'); } const funnelWindowSeconds = funnelWindow * 3600; const funnelWindowMilliseconds = funnelWindowSeconds * 1000; const group = this.getFunnelGroup(funnelGroup); - const funnels = this.getFunnelConditions(events); - const profileFilters = this.getProfileFilters(events); + const funnels = this.getFunnelConditions(eventSeries); + const profileFilters = this.getProfileFilters(eventSeries); const anyFilterOnProfile = profileFilters.length > 0; const anyBreakdownOnProfile = breakdowns.some((b) => b.name.startsWith('profile.'), @@ -152,7 +158,7 @@ export class FunnelService { .where( 'name', 'IN', - events.map((e) => e.name), + eventSeries.map((e) => e.name), ) .groupBy([group[1], ...breakdowns.map((b, index) => `b_${index}`)]); @@ -208,7 +214,7 @@ export class FunnelService { return funnelSeries .map((data) => { - const maxLevel = events.length; + const maxLevel = eventSeries.length; const filledFunnelRes = this.fillFunnel( data.map((d) => ({ level: d.level, count: d.count })), maxLevel, @@ -220,7 +226,7 @@ export class FunnelService { (acc, item, index, list) => { const prev = list[index - 1] ?? { count: totalSessions }; const next = list[index + 1]; - const event = events[item.level - 1]!; + const event = eventSeries[item.level - 1]!; return [ ...acc, { diff --git a/packages/db/src/services/reports.service.ts b/packages/db/src/services/reports.service.ts index e0fab239..1629ee7f 100644 --- a/packages/db/src/services/reports.service.ts +++ b/packages/db/src/services/reports.service.ts @@ -99,7 +99,7 @@ export function transformReport( return { id: report.id, projectId: report.projectId, - events: eventsData.map(transformReportEventItem), + series: eventsData.map(transformReportEventItem), breakdowns: report.breakdowns as IChartBreakdown[], chartType: report.chartType, lineType: (report.lineType as IChartLineType) ?? lineTypes.monotone, diff --git a/packages/trpc/src/routers/chart.helpers.test.ts b/packages/trpc/src/routers/chart.helpers.test.ts deleted file mode 100644 index 31fcb7ff..00000000 --- a/packages/trpc/src/routers/chart.helpers.test.ts +++ /dev/null @@ -1,544 +0,0 @@ -import type { IChartEvent, IChartInput } from '@openpanel/validation'; -import { describe, expect, it } from 'vitest'; -import { withFormula } from './chart.helpers'; - -// Helper to create a test event -function createEvent( - id: string, - name: string, - displayName?: string, -): IChartEvent { - return { - id, - name, - displayName: displayName ?? '', - segment: 'event', - filters: [], - }; -} - -const createChartInput = ( - rest: Pick, -): IChartInput => { - return { - metric: 'sum', - chartType: 'linear', - interval: 'day', - breakdowns: [], - projectId: '1', - startDate: '2025-01-01', - endDate: '2025-01-01', - range: '30d', - previous: false, - formula: '', - ...rest, - }; -}; - -// Helper to create a test series -function createSeries( - name: string[], - event: IChartEvent, - data: Array<{ date: string; count: number }>, -) { - return { - name, - event, - data: data.map((d) => ({ ...d, total_count: d.count })), - }; -} - -describe('withFormula', () => { - describe('edge cases', () => { - it('should return series unchanged when formula is empty', () => { - const events = [createEvent('evt1', 'event1')]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: '', events }), - series, - ); - - expect(result).toEqual(series); - }); - - it('should return series unchanged when series is empty', () => { - const events = [createEvent('evt1', 'event1')]; - const result = withFormula( - createChartInput({ formula: 'A*100', events }), - [], - ); - - expect(result).toEqual([]); - }); - - it('should return series unchanged when series has no data', () => { - const events = [createEvent('evt1', 'event1')]; - const series = [{ name: ['event1'], event: events[0]!, data: [] }]; - - const result = withFormula( - createChartInput({ formula: 'A*100', events }), - series, - ); - - expect(result).toEqual(series); - }); - }); - - describe('single event, no breakdown', () => { - it('should apply simple multiplication formula', () => { - const events = [createEvent('evt1', 'event1')]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - { date: '2025-01-02', count: 20 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A*100', events }), - series, - ); - - expect(result).toHaveLength(1); - expect(result[0]?.data).toEqual([ - { date: '2025-01-01', count: 1000, total_count: 10 }, - { date: '2025-01-02', count: 2000, total_count: 20 }, - ]); - }); - - it('should apply addition formula', () => { - const events = [createEvent('evt1', 'event1')]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 5 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A+10', events }), - series, - ); - - expect(result[0]?.data[0]?.count).toBe(15); - }); - - it('should handle division formula', () => { - const events = [createEvent('evt1', 'event1')]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 100 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/10', events }), - series, - ); - - expect(result[0]?.data[0]?.count).toBe(10); - }); - - it('should handle NaN and Infinity by returning 0', () => { - const events = [createEvent('evt1', 'event1')]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 0 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/0', events }), - series, - ); - - expect(result[0]?.data[0]?.count).toBe(0); - }); - }); - - describe('single event, with breakdown', () => { - it('should apply formula to each breakdown group', () => { - const events = [createEvent('evt1', 'screen_view')]; - const series = [ - createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]), - createSeries(['Android'], events[0]!, [ - { date: '2025-01-01', count: 20 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A*100', events }), - series, - ); - - expect(result).toHaveLength(2); - expect(result[0]?.name).toEqual(['iOS']); - expect(result[0]?.data[0]?.count).toBe(1000); - expect(result[1]?.name).toEqual(['Android']); - expect(result[1]?.data[0]?.count).toBe(2000); - }); - - it('should handle multiple breakdown values', () => { - const events = [createEvent('evt1', 'screen_view')]; - const series = [ - createSeries(['iOS', 'US'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - ]), - createSeries(['Android', 'US'], events[0]!, [ - { date: '2025-01-01', count: 20 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A*2', events }), - series, - ); - - expect(result).toHaveLength(2); - expect(result[0]?.name).toEqual(['iOS', 'US']); - expect(result[0]?.data[0]?.count).toBe(20); - expect(result[1]?.name).toEqual(['Android', 'US']); - expect(result[1]?.data[0]?.count).toBe(40); - }); - }); - - describe('multiple events, no breakdown', () => { - it('should combine two events with division formula', () => { - const events = [ - createEvent('evt1', 'screen_view'), - createEvent('evt2', 'session_start'), - ]; - const series = [ - createSeries(['screen_view'], events[0]!, [ - { date: '2025-01-01', count: 100 }, - ]), - createSeries(['session_start'], events[1]!, [ - { date: '2025-01-01', count: 50 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/B', events }), - series, - ); - - expect(result).toHaveLength(1); - expect(result[0]?.data[0]?.count).toBe(2); - }); - - it('should combine two events with addition formula', () => { - const events = [ - createEvent('evt1', 'event1'), - createEvent('evt2', 'event2'), - ]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - ]), - createSeries(['event2'], events[1]!, [ - { date: '2025-01-01', count: 20 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A+B', events }), - series, - ); - - expect(result[0]?.data[0]?.count).toBe(30); - }); - - it('should handle three events', () => { - const events = [ - createEvent('evt1', 'event1'), - createEvent('evt2', 'event2'), - createEvent('evt3', 'event3'), - ]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - ]), - createSeries(['event2'], events[1]!, [ - { date: '2025-01-01', count: 20 }, - ]), - createSeries(['event3'], events[2]!, [ - { date: '2025-01-01', count: 30 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A+B+C', events }), - series, - ); - - expect(result[0]?.data[0]?.count).toBe(60); - }); - - it('should handle missing data points with 0', () => { - const events = [ - createEvent('evt1', 'event1'), - createEvent('evt2', 'event2'), - ]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - { date: '2025-01-02', count: 20 }, - ]), - createSeries(['event2'], events[1]!, [ - { date: '2025-01-01', count: 5 }, - // Missing 2025-01-02 - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A+B', events }), - series, - ); - - expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5 - expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing) - }); - }); - - describe('multiple events, with breakdown', () => { - it('should match series by breakdown values and apply formula', () => { - const events = [ - createEvent('evt1', 'screen_view'), - createEvent('evt2', 'session_start'), - ]; - const series = [ - // iOS breakdown - createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]), - createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]), - // Android breakdown - createSeries(['Android'], events[0]!, [ - { date: '2025-01-01', count: 200 }, - ]), - createSeries(['Android'], events[1]!, [ - { date: '2025-01-01', count: 100 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/B', events }), - series, - ); - - expect(result).toHaveLength(2); - // iOS: 100/50 = 2 - expect(result[0]?.name).toEqual(['iOS']); - expect(result[0]?.data[0]?.count).toBe(2); - // Android: 200/100 = 2 - expect(result[1]?.name).toEqual(['Android']); - expect(result[1]?.data[0]?.count).toBe(2); - }); - - it('should handle multiple breakdown values matching', () => { - const events = [ - createEvent('evt1', 'screen_view'), - createEvent('evt2', 'session_start'), - ]; - const series = [ - createSeries(['iOS', 'US'], events[0]!, [ - { date: '2025-01-01', count: 100 }, - ]), - createSeries(['iOS', 'US'], events[1]!, [ - { date: '2025-01-01', count: 50 }, - ]), - createSeries(['Android', 'US'], events[0]!, [ - { date: '2025-01-01', count: 200 }, - ]), - createSeries(['Android', 'US'], events[1]!, [ - { date: '2025-01-01', count: 100 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/B', events }), - series, - ); - - expect(result).toHaveLength(2); - expect(result[0]?.name).toEqual(['iOS', 'US']); - expect(result[0]?.data[0]?.count).toBe(2); - expect(result[1]?.name).toEqual(['Android', 'US']); - expect(result[1]?.data[0]?.count).toBe(2); - }); - - it('should handle different date ranges across breakdown groups', () => { - const events = [ - createEvent('evt1', 'screen_view'), - createEvent('evt2', 'session_start'), - ]; - const series = [ - createSeries(['iOS'], events[0]!, [ - { date: '2025-01-01', count: 100 }, - { date: '2025-01-02', count: 200 }, - ]), - createSeries(['iOS'], events[1]!, [ - { date: '2025-01-01', count: 50 }, - { date: '2025-01-02', count: 100 }, - ]), - createSeries(['Android'], events[0]!, [ - { date: '2025-01-01', count: 300 }, - // Missing 2025-01-02 - ]), - createSeries(['Android'], events[1]!, [ - { date: '2025-01-01', count: 150 }, - { date: '2025-01-02', count: 200 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/B', events }), - series, - ); - - expect(result).toHaveLength(2); - // iOS group - expect(result[0]?.name).toEqual(['iOS']); - expect(result[0]?.data[0]?.count).toBe(2); // 100/50 - expect(result[0]?.data[1]?.count).toBe(2); // 200/100 - // Android group - expect(result[1]?.name).toEqual(['Android']); - expect(result[1]?.data[0]?.count).toBe(2); // 300/150 - expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A) - }); - }); - - describe('complex formulas', () => { - it('should handle complex expressions', () => { - const events = [ - createEvent('evt1', 'event1'), - createEvent('evt2', 'event2'), - createEvent('evt3', 'event3'), - ]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - ]), - createSeries(['event2'], events[1]!, [ - { date: '2025-01-01', count: 20 }, - ]), - createSeries(['event3'], events[2]!, [ - { date: '2025-01-01', count: 30 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: '(A+B)*C', events }), - series, - ); - - // (10+20)*30 = 900 - expect(result[0]?.data[0]?.count).toBe(900); - }); - - it('should handle percentage calculations', () => { - const events = [ - createEvent('evt1', 'screen_view'), - createEvent('evt2', 'session_start'), - ]; - const series = [ - createSeries(['screen_view'], events[0]!, [ - { date: '2025-01-01', count: 75 }, - ]), - createSeries(['session_start'], events[1]!, [ - { date: '2025-01-01', count: 100 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: '(A/B)*100', events }), - series, - ); - - // (75/100)*100 = 75 - expect(result[0]?.data[0]?.count).toBe(75); - }); - }); - - describe('error handling', () => { - it('should handle invalid formulas gracefully', () => { - const events = [createEvent('evt1', 'event1')]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'invalid formula', events }), - series, - ); - - // Should return 0 for invalid formulas - expect(result[0]?.data[0]?.count).toBe(0); - }); - - it('should handle division by zero', () => { - const events = [ - createEvent('evt1', 'event1'), - createEvent('evt2', 'event2'), - ]; - const series = [ - createSeries(['event1'], events[0]!, [ - { date: '2025-01-01', count: 10 }, - ]), - createSeries(['event2'], events[1]!, [ - { date: '2025-01-01', count: 0 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/B', events }), - series, - ); - - // Division by zero should result in 0 (Infinity -> 0) - expect(result[0]?.data[0]?.count).toBe(0); - }); - }); - - describe('real-world scenario: article hit ratio', () => { - it('should calculate hit ratio per article path', () => { - const events = [ - createEvent('evt1', 'screen_view'), - createEvent('evt2', 'article_card_seen'), - ]; - const series = [ - // Article 1 - createSeries(['/articles/1'], events[0]!, [ - { date: '2025-01-01', count: 1000 }, - ]), - createSeries(['/articles/1'], events[1]!, [ - { date: '2025-01-01', count: 100 }, - ]), - // Article 2 - createSeries(['/articles/2'], events[0]!, [ - { date: '2025-01-01', count: 500 }, - ]), - createSeries(['/articles/2'], events[1]!, [ - { date: '2025-01-01', count: 200 }, - ]), - ]; - - const result = withFormula( - createChartInput({ formula: 'A/B', events }), - series, - ); - - expect(result).toHaveLength(2); - // Article 1: 1000/100 = 10 - expect(result[0]?.name).toEqual(['/articles/1']); - expect(result[0]?.data[0]?.count).toBe(10); - // Article 2: 500/200 = 2.5 - expect(result[1]?.name).toEqual(['/articles/2']); - expect(result[1]?.data[0]?.count).toBe(2.5); - }); - }); -}); diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts deleted file mode 100644 index f9f1f744..00000000 --- a/packages/trpc/src/routers/chart.helpers.ts +++ /dev/null @@ -1,889 +0,0 @@ -import * as mathjs from 'mathjs'; -import { last, reverse } from 'ramda'; -import sqlstring from 'sqlstring'; - -import type { ISerieDataItem } from '@openpanel/common'; -import { - average, - getPreviousMetric, - groupByLabels, - max, - min, - round, - slug, - sum, -} from '@openpanel/common'; -import { alphabetIds } from '@openpanel/constants'; -import { - TABLE_NAMES, - chQuery, - createSqlBuilder, - formatClickhouseDate, - getChartPrevStartEndDate, - getChartSql, - getChartStartEndDate, - getEventFiltersWhereClause, - getOrganizationSubscriptionChartEndDate, - getSettingsForProject, -} from '@openpanel/db'; -import type { - FinalChart, - IChartEvent, - IChartEventItem, - IChartFormula, - IChartInput, - IChartInputWithDates, - IGetChartDataInput, -} from '@openpanel/validation'; - -export function withFormula( - { formula, events }: IChartInput, - series: Awaited>, -) { - if (!formula) { - return series; - } - - if (!series || series.length === 0) { - return series; - } - - if (!series[0]?.data) { - return series; - } - - // Formulas always use alphabet IDs (A, B, C, etc.), not event IDs - // Group series by breakdown values (the name array) - // This allows us to match series from different events that have the same breakdown values - - // Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names - // When there are breakdowns, name arrays contain breakdown values (not event names) - const hasBreakdowns = series.some( - (serie) => - serie.name.length > 0 && - !events.some((event) => { - if (event.type === 'event') { - return ( - serie.name[0] === event.name || serie.name[0] === event.displayName - ); - } - return false; - }), - ); - - const seriesByBreakdown = new Map(); - - series.forEach((serie) => { - let breakdownKey: string; - - if (hasBreakdowns) { - // With breakdowns: use the entire name array as the breakdown key - // The name array contains breakdown values (e.g., ["iOS"], ["Android"]) - breakdownKey = serie.name.join(':::'); - } else { - // Without breakdowns: group all series together regardless of event name - // This allows formulas to combine multiple events - breakdownKey = ''; - } - - if (!seriesByBreakdown.has(breakdownKey)) { - seriesByBreakdown.set(breakdownKey, []); - } - seriesByBreakdown.get(breakdownKey)!.push(serie); - }); - - // For each breakdown group, apply the formula - const result: typeof series = []; - - for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) { - // Group series by event to ensure we have one series per event - const seriesByEvent = new Map(); - - breakdownSeries.forEach((serie) => { - const eventId = serie.event.id ?? serie.event.name; - // If we already have a series for this event in this breakdown group, skip it - // (shouldn't happen, but just in case) - if (!seriesByEvent.has(eventId)) { - seriesByEvent.set(eventId, serie); - } - }); - - // Get all unique dates across all series in this breakdown group - const allDates = new Set(); - breakdownSeries.forEach((serie) => { - serie.data.forEach((item) => { - allDates.add(item.date); - }); - }); - - // Sort dates chronologically - const sortedDates = Array.from(allDates).sort( - (a, b) => new Date(a).getTime() - new Date(b).getTime(), - ); - - // Apply formula for each date, matching series by event index - const formulaData = sortedDates.map((date) => { - const scope: Record = {}; - - // Build scope using alphabet IDs (A, B, C, etc.) for each event - // This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C") - events.forEach((event, eventIndex) => { - const readableId = alphabetIds[eventIndex]; - if (!readableId) { - throw new Error('no alphabet id for serie in withFormula'); - } - - // Find the series for this event in this breakdown group - // Only events (not formulas) are used in the old formula system - if (event.type !== 'event') { - scope[readableId] = 0; - return; - } - const eventId = event.id ?? event.name; - const matchingSerie = seriesByEvent.get(eventId); - - // Find the data point for this date - // If the series doesn't exist or the date is missing, use 0 - const dataPoint = matchingSerie?.data.find((d) => d.date === date); - scope[readableId] = dataPoint?.count ?? 0; - }); - - // Evaluate the formula with the scope - let count: number; - try { - count = mathjs.parse(formula).compile().evaluate(scope) as number; - } catch (error) { - // If formula evaluation fails, return 0 - count = 0; - } - - return { - date, - count: - Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2), - total_count: breakdownSeries[0]?.data.find((d) => d.date === date) - ?.total_count, - }; - }); - - // Use the first series as a template, but replace its data with formula results - // Preserve the breakdown labels (name array) from the original series - const templateSerie = breakdownSeries[0]!; - result.push({ - ...templateSerie, - data: formulaData, - }); - } - - return result; -} - -function fillFunnel(funnel: { level: number; count: number }[], steps: number) { - const filled = Array.from({ length: steps }, (_, index) => { - const level = index + 1; - const matchingResult = funnel.find((res) => res.level === level); - return { - level, - count: matchingResult ? matchingResult.count : 0, - }; - }); - - // Accumulate counts from top to bottom of the funnel - for (let i = filled.length - 1; i >= 0; i--) { - const step = filled[i]; - const prevStep = filled[i + 1]; - // If there's a previous step, add the count to the current step - if (step && prevStep) { - step.count += prevStep.count; - } - } - return filled.reverse(); -} - -export async function getFunnelData({ - projectId, - startDate, - endDate, - ...payload -}: IChartInput) { - const funnelWindow = (payload.funnelWindow || 24) * 3600; - const funnelGroup = - payload.funnelGroup === 'profile_id' - ? [`COALESCE(nullIf(s.profile_id, ''), e.profile_id)`, 'profile_id'] - : ['session_id', 'session_id']; - - if (!startDate || !endDate) { - throw new Error('startDate and endDate are required'); - } - - if (payload.events.length === 0) { - return { - totalSessions: 0, - steps: [], - }; - } - - const funnels = payload.events - .filter( - (event): event is IChartEventItem & { type: 'event' } => - event.type === 'event', - ) - .map((event) => { - const { sb, getWhere } = createSqlBuilder(); - sb.where = getEventFiltersWhereClause(event.filters); - sb.where.name = `name = ${sqlstring.escape(event.name)}`; - return getWhere().replace('WHERE ', ''); - }); - - const commonWhere = `project_id = ${sqlstring.escape(projectId)} AND - created_at >= '${formatClickhouseDate(startDate)}' AND - created_at <= '${formatClickhouseDate(endDate)}'`; - - // Filter to only events (funnels don't support formulas) - const eventNames = payload.events - .filter((e): e is IChartEventItem & { type: 'event' } => e.type === 'event') - .map((event) => sqlstring.escape(event.name)); - - const innerSql = `SELECT - ${funnelGroup[0]} AS ${funnelGroup[1]}, - windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level - FROM ${TABLE_NAMES.events} e - ${funnelGroup[0] === 'session_id' ? '' : `LEFT JOIN (SELECT profile_id, id FROM sessions WHERE ${commonWhere}) AS s ON s.id = e.session_id`} - WHERE - ${commonWhere} AND - name IN (${eventNames.join(', ')}) - GROUP BY ${funnelGroup[0]}`; - - const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`; - - const funnel = await chQuery<{ level: number; count: number }>(sql); - const maxLevel = payload.events.length; - const filledFunnelRes = fillFunnel(funnel, maxLevel); - - const totalSessions = last(filledFunnelRes)?.count ?? 0; - const steps = reverse(filledFunnelRes).reduce( - (acc, item, index, list) => { - const prev = list[index - 1] ?? { count: totalSessions }; - const eventItem = payload.events[item.level - 1]!; - // Funnels only work with events, not formulas - if (eventItem.type !== 'event') { - return acc; - } - const event = eventItem; - return [ - ...acc, - { - event: { - ...event, - displayName: event.displayName ?? event.name, - }, - count: item.count, - percent: (item.count / totalSessions) * 100, - dropoffCount: prev.count - item.count, - dropoffPercent: 100 - (item.count / prev.count) * 100, - previousCount: prev.count, - }, - ]; - }, - [] as { - event: IChartEvent & { displayName: string }; - count: number; - percent: number; - dropoffCount: number; - dropoffPercent: number; - previousCount: number; - }[], - ); - - return { - totalSessions, - steps, - }; -} - -export async function getChartSerie( - payload: IGetChartDataInput, - timezone: string, -) { - let result = await chQuery( - getChartSql({ ...payload, timezone }), - { - session_timezone: timezone, - }, - ); - - if (result.length === 0 && payload.breakdowns.length > 0) { - result = await chQuery( - getChartSql({ - ...payload, - breakdowns: [], - timezone, - }), - { - session_timezone: timezone, - }, - ); - } - - return groupByLabels(result).map((serie) => { - return { - ...serie, - event: payload.event, - }; - }); -} - -// Normalize events to ensure they have a type field -function normalizeEventItem( - item: IChartEventItem | IChartEvent, -): IChartEventItem { - if ('type' in item) { - return item; - } - // Old format without type field - assume it's an event - return { ...item, type: 'event' as const }; -} - -// Calculate formula result from previous series -function calculateFormulaSeries( - formula: IChartFormula, - previousSeries: Awaited>, - normalizedEvents: IChartEventItem[], - formulaIndex: number, -): Awaited> { - if (!previousSeries || previousSeries.length === 0) { - return []; - } - - if (!previousSeries[0]?.data) { - return []; - } - - // Detect if we have breakdowns by checking if series names contain breakdown values - // (not event/formula names) - const hasBreakdowns = previousSeries.some( - (serie) => - serie.name.length > 1 || // Multiple name parts = breakdowns - (serie.name.length === 1 && - !normalizedEvents - .slice(0, formulaIndex) - .some( - (event) => - event.type === 'event' && - (serie.name[0] === event.name || - serie.name[0] === event.displayName), - ) && - !normalizedEvents - .slice(0, formulaIndex) - .some( - (event) => - event.type === 'formula' && - (serie.name[0] === event.displayName || - serie.name[0] === event.formula), - )), - ); - - const seriesByBreakdown = new Map< - string, - Awaited> - >(); - - previousSeries.forEach((serie) => { - let breakdownKey: string; - - if (hasBreakdowns) { - // With breakdowns: use the entire name array as the breakdown key - // Skip the first element (event/formula name) and use breakdown values - breakdownKey = serie.name.slice(1).join(':::'); - } else { - // Without breakdowns: group all series together - // This allows formulas to combine multiple events/formulas - breakdownKey = ''; - } - - if (!seriesByBreakdown.has(breakdownKey)) { - seriesByBreakdown.set(breakdownKey, []); - } - seriesByBreakdown.get(breakdownKey)!.push(serie); - }); - - const result: Awaited> = []; - - for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) { - // Group series by event index to ensure we have one series per event - const seriesByEventIndex = new Map< - number, - (typeof previousSeries)[number] - >(); - - breakdownSeries.forEach((serie) => { - // Find which event index this series belongs to - const eventIndex = normalizedEvents - .slice(0, formulaIndex) - .findIndex((event) => { - if (event.type === 'event') { - const eventId = event.id ?? event.name; - return ( - serie.event.id === eventId || serie.event.name === event.name - ); - } - return false; - }); - - if (eventIndex >= 0 && !seriesByEventIndex.has(eventIndex)) { - seriesByEventIndex.set(eventIndex, serie); - } - }); - - // Get all unique dates across all series in this breakdown group - const allDates = new Set(); - breakdownSeries.forEach((serie) => { - serie.data.forEach((item) => { - allDates.add(item.date); - }); - }); - - // Sort dates chronologically - const sortedDates = Array.from(allDates).sort( - (a, b) => new Date(a).getTime() - new Date(b).getTime(), - ); - - // Apply formula for each date - const formulaData = sortedDates.map((date) => { - const scope: Record = {}; - - // Build scope using alphabet IDs (A, B, C, etc.) for each event before this formula - normalizedEvents.slice(0, formulaIndex).forEach((event, eventIndex) => { - const readableId = alphabetIds[eventIndex]; - if (!readableId) { - return; - } - - if (event.type === 'event') { - const matchingSerie = seriesByEventIndex.get(eventIndex); - const dataPoint = matchingSerie?.data.find((d) => d.date === date); - scope[readableId] = dataPoint?.count ?? 0; - } else { - // If it's a formula, we need to get its calculated value - // This handles nested formulas - const formulaSerie = breakdownSeries.find( - (s) => s.event.id === event.id, - ); - const dataPoint = formulaSerie?.data.find((d) => d.date === date); - scope[readableId] = dataPoint?.count ?? 0; - } - }); - - // Evaluate the formula with the scope - let count: number; - try { - count = mathjs - .parse(formula.formula) - .compile() - .evaluate(scope) as number; - } catch (error) { - count = 0; - } - - return { - date, - count: - Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2), - total_count: breakdownSeries[0]?.data.find((d) => d.date === date) - ?.total_count, - }; - }); - - // Use the first series as a template - const templateSerie = breakdownSeries[0]!; - - // For formulas, construct the name array: - // - Without breakdowns: use formula displayName/formula - // - With breakdowns: use formula displayName/formula as first element, then breakdown values - let formulaName: string[]; - if (hasBreakdowns) { - // With breakdowns: formula name + breakdown values (skip first element which is event/formula name) - const formulaDisplayName = formula.displayName || formula.formula; - formulaName = [formulaDisplayName, ...templateSerie.name.slice(1)]; - } else { - // Without breakdowns: just formula name - formulaName = [formula.displayName || formula.formula]; - } - - result.push({ - ...templateSerie, - name: formulaName, - // For formulas, create a simplified event object - // We use 'as' because formulas don't have segment/filters, but the event - // object is only used for id/name lookups later, so this is safe - event: { - id: formula.id, - name: formula.displayName || formula.formula, - displayName: formula.displayName, - segment: 'event' as const, - filters: [], - } as IChartEvent, - data: formulaData, - }); - } - - return result; -} - -export type IGetChartSerie = Awaited>[number]; -export async function getChartSeries( - input: IChartInputWithDates, - timezone: string, -) { - // Normalize all events to have type field - const normalizedEvents = input.events.map(normalizeEventItem); - - // Process events sequentially - events fetch data, formulas calculate from previous series - const allSeries: Awaited> = []; - - for (let i = 0; i < normalizedEvents.length; i++) { - const item = normalizedEvents[i]!; - - if (item.type === 'event') { - // Fetch data for event - const eventSeries = await getChartSerie( - { - ...input, - event: item, - }, - timezone, - ); - allSeries.push(...eventSeries); - } else if (item.type === 'formula') { - // Calculate formula from previous series - const formulaSeries = calculateFormulaSeries( - item, - allSeries, - normalizedEvents, - i, - ); - allSeries.push(...formulaSeries); - } - } - - // Apply top-level formula if present (for backward compatibility) - try { - if (input.formula) { - return withFormula(input, allSeries); - } - } catch (e) { - // If formula evaluation fails, return series as-is - } - - return allSeries; -} - -export async function getChart(input: IChartInput) { - const { timezone } = await getSettingsForProject(input.projectId); - const currentPeriod = getChartStartEndDate(input, timezone); - const previousPeriod = getChartPrevStartEndDate(currentPeriod); - - const endDate = await getOrganizationSubscriptionChartEndDate( - input.projectId, - currentPeriod.endDate, - ); - - if (endDate) { - currentPeriod.endDate = endDate; - } - - const promises = [getChartSeries({ ...input, ...currentPeriod }, timezone)]; - - if (input.previous) { - promises.push( - getChartSeries( - { - ...input, - ...previousPeriod, - }, - timezone, - ), - ); - } - - // Normalize events for consistent handling - const normalizedEvents = input.events.map(normalizeEventItem); - - const getSerieId = (serie: IGetChartSerie) => - [slug(serie.name.join('-')), serie.event.id].filter(Boolean).join('-'); - const result = await Promise.all(promises); - const series = result[0]!; - const previousSeries = result[1]; - const limit = input.limit || 300; - const offset = input.offset || 0; - const includeEventAlphaId = normalizedEvents.length > 1; - - // Calculate metrics cache for formulas - // Map> - const metricsCache = new Map< - number, - Map< - string, - { - sum: number; - average: number; - min: number; - max: number; - count: number; - } - > - >(); - - // Initialize cache - for (let i = 0; i < normalizedEvents.length; i++) { - metricsCache.set(i, new Map()); - } - - // First pass: calculate standard metrics for all series and populate cache - // We iterate through series in order, but since series array is flattened, we need to be careful. - // Fortunately, events are processed sequentially, so dependencies usually appear before formulas. - // However, to be safe, we'll compute metrics for all series first. - - const seriesWithMetrics = series.map((serie) => { - // Find the index of the event/formula that produced this series - const eventIndex = normalizedEvents.findIndex((event) => { - if (event.type === 'event') { - return event.id === serie.event.id || event.name === serie.event.name; - } - return event.id === serie.event.id; - }); - - const standardMetrics = { - sum: sum(serie.data.map((item) => item.count)), - average: round(average(serie.data.map((item) => item.count)), 2), - min: min(serie.data.map((item) => item.count)), - max: max(serie.data.map((item) => item.count)), - count: serie.data.find((item) => !!item.total_count)?.total_count || 0, - }; - - // Store in cache - if (eventIndex >= 0) { - const breakdownSignature = serie.name.slice(1).join(':::'); - metricsCache.get(eventIndex)?.set(breakdownSignature, standardMetrics); - } - - return { - serie, - eventIndex, - metrics: standardMetrics, - }; - }); - - // Second pass: Re-calculate metrics for formulas using dependency metrics - // We iterate through normalizedEvents to process in dependency order - normalizedEvents.forEach((event, eventIndex) => { - if (event.type !== 'formula') return; - - // We dont have count on formulas so use sum instead - const property = 'count'; - // Iterate through all series corresponding to this formula - seriesWithMetrics.forEach((item) => { - if (item.eventIndex !== eventIndex) return; - - const breakdownSignature = item.serie.name.slice(1).join(':::'); - const scope: Record = {}; - - // Build scope from dependency metrics - normalizedEvents.slice(0, eventIndex).forEach((depEvent, depIndex) => { - const readableId = alphabetIds[depIndex]; - if (!readableId) return; - - // Get metric from cache for the dependency with the same breakdown signature - const depMetrics = metricsCache.get(depIndex)?.get(breakdownSignature); - // Use sum as the default metric for formula calculation on totals - scope[readableId] = depMetrics?.[property] ?? 0; - }); - - // Evaluate formula - let calculatedSum: number; - try { - calculatedSum = mathjs - .parse(event.formula) - .compile() - .evaluate(scope) as number; - } catch (error) { - calculatedSum = 0; - } - - // Update metrics with calculated sum - // For formulas, the "sum" metric (Total) should be the result of the formula applied to the totals - // The "average" metric usually remains average of data points, or calculatedSum / intervals - item.metrics = { - ...item.metrics, - [property]: - Number.isNaN(calculatedSum) || !Number.isFinite(calculatedSum) - ? 0 - : round(calculatedSum, 2), - }; - - // Update cache with new metrics so dependent formulas can use it - metricsCache.get(eventIndex)?.set(breakdownSignature, item.metrics); - }); - }); - - const final: FinalChart = { - series: seriesWithMetrics.map(({ serie, eventIndex, metrics }) => { - const alphaId = alphabetIds[eventIndex]; - const previousSerie = previousSeries?.find( - (prevSerie) => getSerieId(prevSerie) === getSerieId(serie), - ); - - // Determine if this is a formula series - const isFormula = normalizedEvents[eventIndex]?.type === 'formula'; - const eventItem = normalizedEvents[eventIndex]; - - const event = { - id: serie.event.id, - name: serie.event.displayName || serie.event.name, - }; - - // Construct names array based on whether it's a formula or event - let names: string[]; - if (isFormula && eventItem?.type === 'formula') { - // For formulas: - // - Without breakdowns: use displayName/formula (with optional alpha ID) - // - With breakdowns: use displayName/formula + breakdown values (with optional alpha ID) - const formulaDisplayName = eventItem.displayName || eventItem.formula; - if (input.breakdowns.length === 0) { - // No breakdowns: just formula name - names = includeEventAlphaId - ? [`(${alphaId}) ${formulaDisplayName}`] - : [formulaDisplayName]; - } else { - // With breakdowns: formula name + breakdown values - names = includeEventAlphaId - ? [`(${alphaId}) ${formulaDisplayName}`, ...serie.name.slice(1)] - : [formulaDisplayName, ...serie.name.slice(1)]; - } - } else { - // For events: use existing logic - names = - input.breakdowns.length === 0 && serie.event.displayName - ? [serie.event.displayName] - : includeEventAlphaId - ? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)] - : serie.name; - } - - return { - id: getSerieId(serie), - names, - event, - metrics: { - ...metrics, - ...(input.previous - ? { - previous: { - sum: getPreviousMetric( - metrics.sum, - previousSerie - ? sum(previousSerie?.data.map((item) => item.count)) - : null, - ), - average: getPreviousMetric( - metrics.average, - previousSerie - ? round( - average( - previousSerie?.data.map((item) => item.count), - ), - 2, - ) - : null, - ), - min: getPreviousMetric( - metrics.sum, - previousSerie - ? min(previousSerie?.data.map((item) => item.count)) - : null, - ), - max: getPreviousMetric( - metrics.sum, - previousSerie - ? max(previousSerie?.data.map((item) => item.count)) - : null, - ), - count: getPreviousMetric( - metrics.count ?? 0, - previousSerie?.data[0]?.total_count ?? null, - ), - }, - } - : {}), - }, - data: serie.data.map((item, index) => ({ - date: item.date, - count: item.count ?? 0, - previous: previousSerie?.data[index] - ? getPreviousMetric( - item.count ?? 0, - previousSerie?.data[index]?.count ?? null, - ) - : undefined, - })), - }; - }), - metrics: { - sum: 0, - average: 0, - min: 0, - max: 0, - count: undefined, - }, - }; - - // Sort by sum - final.series = final.series - .sort((a, b) => { - if (input.chartType === 'linear') { - const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0); - const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0); - return sumB - sumA; - } - return (b.metrics[input.metric] ?? 0) - (a.metrics[input.metric] ?? 0); - }) - .slice(offset, limit ? offset + limit : series.length); - - final.metrics.sum = sum(final.series.map((item) => item.metrics.sum)); - final.metrics.average = round( - average(final.series.map((item) => item.metrics.average)), - 2, - ); - final.metrics.min = min(final.series.map((item) => item.metrics.min)); - final.metrics.max = max(final.series.map((item) => item.metrics.max)); - if (input.previous) { - final.metrics.previous = { - sum: getPreviousMetric( - final.metrics.sum, - sum(final.series.map((item) => item.metrics.previous?.sum?.value ?? 0)), - ), - average: getPreviousMetric( - final.metrics.average, - round( - average( - final.series.map( - (item) => item.metrics.previous?.average?.value ?? 0, - ), - ), - 2, - ), - ), - min: getPreviousMetric( - final.metrics.min, - min(final.series.map((item) => item.metrics.previous?.min?.value ?? 0)), - ), - max: getPreviousMetric( - final.metrics.max, - max(final.series.map((item) => item.metrics.previous?.max?.value ?? 0)), - ), - count: undefined, - }; - } - - return final; -} diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 4932d68a..44f9575b 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -24,13 +24,16 @@ import { } from '@openpanel/db'; import { zChartEvent, + zChartEventFilter, zChartInput, + zChartSeries, zCriteria, zRange, zTimeInterval, } from '@openpanel/validation'; import { round } from '@openpanel/common'; +import { ChartEngine } from '@openpanel/db'; import { differenceInDays, differenceInMonths, @@ -45,7 +48,6 @@ import { protectedProcedure, publicProcedure, } from '../trpc'; -import { getChart } from './chart.helpers'; function utc(date: string | Date) { if (typeof date === 'string') { @@ -407,7 +409,8 @@ export const chartRouter = createTRPCRouter({ } } - return getChart(input); + // Use new chart engine + return ChartEngine.execute(input); }), cohort: protectedProcedure .input( @@ -542,151 +545,74 @@ export const chartRouter = createTRPCRouter({ .input( z.object({ projectId: z.string(), - event: zChartEvent, date: z.string().describe('The date for the data point (ISO string)'), - breakdowns: z - .array( - z.object({ - id: z.string().optional(), - name: z.string(), - }), - ) - .default([]), interval: zTimeInterval.default('day'), - startDate: z.string(), - endDate: z.string(), - filters: z - .array( - z.object({ - id: z.string().optional(), - name: z.string(), - operator: z.string(), - value: z.array( - z.union([z.string(), z.number(), z.boolean(), z.null()]), - ), - }), - ) - .default([]), - limit: z.number().default(100), + series: zChartSeries, + breakdowns: z.record(z.string(), z.string()).optional(), }), ) .query(async ({ input }) => { const { timezone } = await getSettingsForProject(input.projectId); - const { - projectId, - event, - date, - breakdowns, - interval, - startDate, - endDate, - filters, - limit, - } = input; + const { projectId, date, series } = input; + const limit = 100; + const serie = series[0]; + + if (!serie) { + throw new Error('Series not found'); + } + + if (serie.type !== 'event') { + throw new Error('Series must be an event'); + } // Build the date range for the specific interval bucket const dateObj = new Date(date); - let bucketStart: Date; - let bucketEnd: Date; - - switch (interval) { - case 'minute': - bucketStart = new Date( - dateObj.getFullYear(), - dateObj.getMonth(), - dateObj.getDate(), - dateObj.getHours(), - dateObj.getMinutes(), - ); - bucketEnd = new Date(bucketStart.getTime() + 60 * 1000); - break; - case 'hour': - bucketStart = new Date( - dateObj.getFullYear(), - dateObj.getMonth(), - dateObj.getDate(), - dateObj.getHours(), - ); - bucketEnd = new Date(bucketStart.getTime() + 60 * 60 * 1000); - break; - case 'day': - bucketStart = new Date( - dateObj.getFullYear(), - dateObj.getMonth(), - dateObj.getDate(), - ); - bucketEnd = new Date(bucketStart.getTime() + 24 * 60 * 60 * 1000); - break; - case 'week': - bucketStart = new Date(dateObj); - bucketStart.setDate(dateObj.getDate() - dateObj.getDay()); - bucketStart.setHours(0, 0, 0, 0); - bucketEnd = new Date(bucketStart.getTime() + 7 * 24 * 60 * 60 * 1000); - break; - case 'month': - bucketStart = new Date(dateObj.getFullYear(), dateObj.getMonth(), 1); - bucketEnd = new Date( - dateObj.getFullYear(), - dateObj.getMonth() + 1, - 1, - ); - break; - default: - bucketStart = new Date( - dateObj.getFullYear(), - dateObj.getMonth(), - dateObj.getDate(), - ); - bucketEnd = new Date(bucketStart.getTime() + 24 * 60 * 60 * 1000); - } - // Build query to get unique profile_ids for this time bucket - const { sb, join, getWhere, getFrom, getJoins } = createSqlBuilder(); + const { sb, getSql } = createSqlBuilder(); - sb.where = getEventFiltersWhereClause([...event.filters, ...filters]); + sb.select.profile_id = 'DISTINCT profile_id'; + sb.where = getEventFiltersWhereClause(serie.filters); sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`; - sb.where.dateRange = `created_at >= '${formatClickhouseDate(bucketStart.toISOString())}' AND created_at < '${formatClickhouseDate(bucketEnd.toISOString())}'`; - - if (event.name !== '*') { - sb.where.eventName = `name = ${sqlstring.escape(event.name)}`; + sb.where.dateRange = `${clix.toStartOf('created_at', input.interval)} = ${clix.toDate(sqlstring.escape(formatClickhouseDate(dateObj)), input.interval)}`; + if (serie.name !== '*') { + sb.where.eventName = `name = ${sqlstring.escape(serie.name)}`; } - // Handle breakdowns if provided - const anyBreakdownOnProfile = breakdowns.some((breakdown) => - breakdown.name.startsWith('profile.'), - ); - const anyFilterOnProfile = [...event.filters, ...filters].some((filter) => - filter.name.startsWith('profile.'), - ); - - if (anyFilterOnProfile || anyBreakdownOnProfile) { - sb.joins.profiles = `LEFT ANY JOIN (SELECT - id as "profile.id", - email as "profile.email", - first_name as "profile.first_name", - last_name as "profile.last_name", - properties as "profile.properties" - FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`; + console.log('> breakdowns', input.breakdowns); + if (input.breakdowns) { + Object.entries(input.breakdowns).forEach(([key, value]) => { + sb.where[`breakdown_${key}`] = `${key} = ${sqlstring.escape(value)}`; + }); } + // // Handle breakdowns if provided + // const anyBreakdownOnProfile = breakdowns.some((breakdown) => + // breakdown.name.startsWith('profile.'), + // ); + // const anyFilterOnProfile = [...event.filters, ...filters].some((filter) => + // filter.name.startsWith('profile.'), + // ); + + // if (anyFilterOnProfile || anyBreakdownOnProfile) { + // sb.joins.profiles = `LEFT ANY JOIN (SELECT + // id as "profile.id", + // email as "profile.email", + // first_name as "profile.first_name", + // last_name as "profile.last_name", + // properties as "profile.properties" + // FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`; + // } + // Apply breakdown filters if provided - breakdowns.forEach((breakdown) => { - // This is simplified - in reality we'd need to match the breakdown value - // For now, we'll just get all profiles for the time bucket - }); + // breakdowns.forEach((breakdown) => { + // // This is simplified - in reality we'd need to match the breakdown value + // // For now, we'll just get all profiles for the time bucket + // }); // Get unique profile IDs - const profileIdsQuery = ` - SELECT DISTINCT profile_id - FROM ${TABLE_NAMES.events} - ${getJoins()} - WHERE ${join(sb.where, ' AND ')} - AND profile_id != '' - LIMIT ${limit} - `; - - const profileIds = await chQuery<{ profile_id: string }>(profileIdsQuery); - + console.log('profileIdsQuery', getSql()); + const profileIds = await chQuery<{ profile_id: string }>(getSql()); + console.log('profileIds', profileIds.length); if (profileIds.length === 0) { return []; } diff --git a/packages/trpc/src/routers/report.ts b/packages/trpc/src/routers/report.ts index d4ae4cef..b17acb8c 100644 --- a/packages/trpc/src/routers/report.ts +++ b/packages/trpc/src/routers/report.ts @@ -46,7 +46,7 @@ export const reportRouter = createTRPCRouter({ projectId: dashboard.projectId, dashboardId, name: report.name, - events: report.events, + events: report.series, interval: report.interval, breakdowns: report.breakdowns, chartType: report.chartType, @@ -91,7 +91,7 @@ export const reportRouter = createTRPCRouter({ }, data: { name: report.name, - events: report.events, + events: report.series, interval: report.interval, breakdowns: report.breakdowns, chartType: report.chartType, diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index 45a99ff7..bc8777ee 100644 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -88,9 +88,21 @@ export const zChartBreakdown = z.object({ // Support both old format (array of events without type) and new format (array of event/formula items) // Preprocess to normalize: if item has 'type' field, use discriminated union; otherwise, add type: 'event' -export const zChartEvents = z.preprocess((val) => { - if (!Array.isArray(val)) return val; - return val.map((item: any) => { +export const zChartSeries = z.preprocess((val) => { + if (!val) return val; + let processedVal = val; + + // If the input is an object with numeric keys, convert it to an array + if (typeof val === 'object' && val !== null && !Array.isArray(val)) { + const keys = Object.keys(val).sort( + (a, b) => Number.parseInt(a) - Number.parseInt(b), + ); + processedVal = keys.map((key) => (val as any)[key]); + } + + if (!Array.isArray(processedVal)) return processedVal; + + return processedVal.map((item: any) => { // If item already has type field, return as-is if (item && typeof item === 'object' && 'type' in item) { return item; @@ -101,7 +113,14 @@ export const zChartEvents = z.preprocess((val) => { } return item; }); -}, z.array(zChartEventItem)); +}, z + .array(zChartEventItem) + .describe( + 'Array of series (events or formulas) to be tracked and displayed in the chart', + )); + +// Keep zChartEvents as an alias for backward compatibility during migration +export const zChartEvents = zChartSeries; export const zChartBreakdowns = z.array(zChartBreakdown); export const zChartType = z.enum(objectToZodEnums(chartTypes)); @@ -116,7 +135,7 @@ export const zRange = z.enum(objectToZodEnums(timeWindows)); export const zCriteria = z.enum(['on_or_after', 'on']); -export const zChartInput = z.object({ +export const zChartInputBase = z.object({ chartType: zChartType .default('linear') .describe('What type of chart should be displayed'), @@ -125,8 +144,8 @@ export const zChartInput = z.object({ .describe( 'The time interval for data aggregation (e.g., day, week, month)', ), - events: zChartEvents.describe( - 'Array of events to be tracked and displayed in the chart', + series: zChartSeries.describe( + 'Array of series (events or formulas) to be tracked and displayed in the chart', ), breakdowns: zChartBreakdowns .default([]) @@ -183,7 +202,15 @@ export const zChartInput = z.object({ .describe('Time window in hours for funnel analysis'), }); -export const zReportInput = zChartInput.extend({ +export const zChartInput = z.preprocess((val) => { + if (val && typeof val === 'object' && 'events' in val && !('series' in val)) { + // Migrate old 'events' field to 'series' + return { ...val, series: val.events }; + } + return val; +}, zChartInputBase); + +export const zReportInput = zChartInputBase.extend({ name: z.string().describe('The user-defined name for the report'), lineType: zLineType.describe('The visual style of the line in the chart'), unit: z diff --git a/packages/validation/src/types.validation.ts b/packages/validation/src/types.validation.ts index a0ccf27f..5f6db611 100644 --- a/packages/validation/src/types.validation.ts +++ b/packages/validation/src/types.validation.ts @@ -8,6 +8,7 @@ import type { zChartFormula, zChartInput, zChartInputAI, + zChartSeries, zChartType, zCriteria, zLineType, @@ -28,6 +29,9 @@ export type IChartProps = z.infer & { export type IChartEvent = z.infer; export type IChartFormula = z.infer; export type IChartEventItem = z.infer; +export type IChartSeries = z.infer; +// Backward compatibility alias +export type IChartEvents = IChartSeries; export type IChartEventSegment = z.infer; export type IChartEventFilter = IChartEvent['filters'][number]; export type IChartEventFilterValue = @@ -49,7 +53,7 @@ export type IGetChartDataInput = { projectId: string; startDate: string; endDate: string; -} & Omit; +} & Omit; export type ICriteria = z.infer; export type PreviousValue = @@ -81,6 +85,7 @@ export type IChartSerie = { event: { id?: string; name: string; + breakdowns?: Record; }; metrics: Metrics; data: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a48c83b6..2017db09 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -629,9 +629,6 @@ importers: lucide-react: specifier: ^0.476.0 version: 0.476.0(react@19.1.1) - mathjs: - specifier: ^12.3.2 - version: 12.3.2 mitt: specifier: ^3.0.1 version: 3.0.1 @@ -1071,6 +1068,9 @@ importers: jiti: specifier: ^2.4.1 version: 2.4.1 + mathjs: + specifier: ^12.3.2 + version: 12.3.2 prisma-json-types-generator: specifier: ^3.1.1 version: 3.1.1(prisma@6.14.0(magicast@0.3.5)(typescript@5.9.3))(typescript@5.9.3) @@ -10313,9 +10313,6 @@ packages: decimal.js-light@2.5.1: resolution: {integrity: sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==} - decimal.js@10.4.3: - resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} - decimal.js@10.6.0: resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==} @@ -26288,8 +26285,6 @@ snapshots: decimal.js-light@2.5.1: {} - decimal.js@10.4.3: {} - decimal.js@10.6.0: {} decode-named-character-reference@1.0.2: @@ -29307,7 +29302,7 @@ snapshots: dependencies: '@babel/runtime': 7.23.9 complex.js: 2.1.1 - decimal.js: 10.4.3 + decimal.js: 10.6.0 escape-latex: 1.2.0 fraction.js: 4.3.4 javascript-natural-sort: 0.7.1