diff --git a/packages/trpc/package.json b/packages/trpc/package.json index 6632ce43..e5ade19c 100644 --- a/packages/trpc/package.json +++ b/packages/trpc/package.json @@ -4,6 +4,7 @@ "type": "module", "main": "index.ts", "scripts": { + "test": "vitest", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/packages/trpc/src/routers/chart.helpers.test.ts b/packages/trpc/src/routers/chart.helpers.test.ts new file mode 100644 index 00000000..31fcb7ff --- /dev/null +++ b/packages/trpc/src/routers/chart.helpers.test.ts @@ -0,0 +1,544 @@ +import type { IChartEvent, IChartInput } from '@openpanel/validation'; +import { describe, expect, it } from 'vitest'; +import { withFormula } from './chart.helpers'; + +// Helper to create a test event +function createEvent( + id: string, + name: string, + displayName?: string, +): IChartEvent { + return { + id, + name, + displayName: displayName ?? '', + segment: 'event', + filters: [], + }; +} + +const createChartInput = ( + rest: Pick, +): IChartInput => { + return { + metric: 'sum', + chartType: 'linear', + interval: 'day', + breakdowns: [], + projectId: '1', + startDate: '2025-01-01', + endDate: '2025-01-01', + range: '30d', + previous: false, + formula: '', + ...rest, + }; +}; + +// Helper to create a test series +function createSeries( + name: string[], + event: IChartEvent, + data: Array<{ date: string; count: number }>, +) { + return { + name, + event, + data: data.map((d) => ({ ...d, total_count: d.count })), + }; +} + +describe('withFormula', () => { + describe('edge cases', () => { + it('should return series unchanged when formula is empty', () => { + const events = [createEvent('evt1', 'event1')]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: '', events }), + series, + ); + + expect(result).toEqual(series); + }); + + it('should return series unchanged when series is empty', () => { + const events = [createEvent('evt1', 'event1')]; + const result = withFormula( + createChartInput({ formula: 'A*100', events }), + [], + ); + + expect(result).toEqual([]); + }); + + it('should return series unchanged when series has no data', () => { + const events = [createEvent('evt1', 'event1')]; + const series = [{ name: ['event1'], event: events[0]!, data: [] }]; + + const result = withFormula( + createChartInput({ formula: 'A*100', events }), + series, + ); + + expect(result).toEqual(series); + }); + }); + + describe('single event, no breakdown', () => { + it('should apply simple multiplication formula', () => { + const events = [createEvent('evt1', 'event1')]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + { date: '2025-01-02', count: 20 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A*100', events }), + series, + ); + + expect(result).toHaveLength(1); + expect(result[0]?.data).toEqual([ + { date: '2025-01-01', count: 1000, total_count: 10 }, + { date: '2025-01-02', count: 2000, total_count: 20 }, + ]); + }); + + it('should apply addition formula', () => { + const events = [createEvent('evt1', 'event1')]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 5 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A+10', events }), + series, + ); + + expect(result[0]?.data[0]?.count).toBe(15); + }); + + it('should handle division formula', () => { + const events = [createEvent('evt1', 'event1')]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 100 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/10', events }), + series, + ); + + expect(result[0]?.data[0]?.count).toBe(10); + }); + + it('should handle NaN and Infinity by returning 0', () => { + const events = [createEvent('evt1', 'event1')]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 0 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/0', events }), + series, + ); + + expect(result[0]?.data[0]?.count).toBe(0); + }); + }); + + describe('single event, with breakdown', () => { + it('should apply formula to each breakdown group', () => { + const events = [createEvent('evt1', 'screen_view')]; + const series = [ + createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]), + createSeries(['Android'], events[0]!, [ + { date: '2025-01-01', count: 20 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A*100', events }), + series, + ); + + expect(result).toHaveLength(2); + expect(result[0]?.name).toEqual(['iOS']); + expect(result[0]?.data[0]?.count).toBe(1000); + expect(result[1]?.name).toEqual(['Android']); + expect(result[1]?.data[0]?.count).toBe(2000); + }); + + it('should handle multiple breakdown values', () => { + const events = [createEvent('evt1', 'screen_view')]; + const series = [ + createSeries(['iOS', 'US'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + ]), + createSeries(['Android', 'US'], events[0]!, [ + { date: '2025-01-01', count: 20 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A*2', events }), + series, + ); + + expect(result).toHaveLength(2); + expect(result[0]?.name).toEqual(['iOS', 'US']); + expect(result[0]?.data[0]?.count).toBe(20); + expect(result[1]?.name).toEqual(['Android', 'US']); + expect(result[1]?.data[0]?.count).toBe(40); + }); + }); + + describe('multiple events, no breakdown', () => { + it('should combine two events with division formula', () => { + const events = [ + createEvent('evt1', 'screen_view'), + createEvent('evt2', 'session_start'), + ]; + const series = [ + createSeries(['screen_view'], events[0]!, [ + { date: '2025-01-01', count: 100 }, + ]), + createSeries(['session_start'], events[1]!, [ + { date: '2025-01-01', count: 50 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/B', events }), + series, + ); + + expect(result).toHaveLength(1); + expect(result[0]?.data[0]?.count).toBe(2); + }); + + it('should combine two events with addition formula', () => { + const events = [ + createEvent('evt1', 'event1'), + createEvent('evt2', 'event2'), + ]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + ]), + createSeries(['event2'], events[1]!, [ + { date: '2025-01-01', count: 20 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A+B', events }), + series, + ); + + expect(result[0]?.data[0]?.count).toBe(30); + }); + + it('should handle three events', () => { + const events = [ + createEvent('evt1', 'event1'), + createEvent('evt2', 'event2'), + createEvent('evt3', 'event3'), + ]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + ]), + createSeries(['event2'], events[1]!, [ + { date: '2025-01-01', count: 20 }, + ]), + createSeries(['event3'], events[2]!, [ + { date: '2025-01-01', count: 30 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A+B+C', events }), + series, + ); + + expect(result[0]?.data[0]?.count).toBe(60); + }); + + it('should handle missing data points with 0', () => { + const events = [ + createEvent('evt1', 'event1'), + createEvent('evt2', 'event2'), + ]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + { date: '2025-01-02', count: 20 }, + ]), + createSeries(['event2'], events[1]!, [ + { date: '2025-01-01', count: 5 }, + // Missing 2025-01-02 + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A+B', events }), + series, + ); + + expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5 + expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing) + }); + }); + + describe('multiple events, with breakdown', () => { + it('should match series by breakdown values and apply formula', () => { + const events = [ + createEvent('evt1', 'screen_view'), + createEvent('evt2', 'session_start'), + ]; + const series = [ + // iOS breakdown + createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]), + createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]), + // Android breakdown + createSeries(['Android'], events[0]!, [ + { date: '2025-01-01', count: 200 }, + ]), + createSeries(['Android'], events[1]!, [ + { date: '2025-01-01', count: 100 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/B', events }), + series, + ); + + expect(result).toHaveLength(2); + // iOS: 100/50 = 2 + expect(result[0]?.name).toEqual(['iOS']); + expect(result[0]?.data[0]?.count).toBe(2); + // Android: 200/100 = 2 + expect(result[1]?.name).toEqual(['Android']); + expect(result[1]?.data[0]?.count).toBe(2); + }); + + it('should handle multiple breakdown values matching', () => { + const events = [ + createEvent('evt1', 'screen_view'), + createEvent('evt2', 'session_start'), + ]; + const series = [ + createSeries(['iOS', 'US'], events[0]!, [ + { date: '2025-01-01', count: 100 }, + ]), + createSeries(['iOS', 'US'], events[1]!, [ + { date: '2025-01-01', count: 50 }, + ]), + createSeries(['Android', 'US'], events[0]!, [ + { date: '2025-01-01', count: 200 }, + ]), + createSeries(['Android', 'US'], events[1]!, [ + { date: '2025-01-01', count: 100 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/B', events }), + series, + ); + + expect(result).toHaveLength(2); + expect(result[0]?.name).toEqual(['iOS', 'US']); + expect(result[0]?.data[0]?.count).toBe(2); + expect(result[1]?.name).toEqual(['Android', 'US']); + expect(result[1]?.data[0]?.count).toBe(2); + }); + + it('should handle different date ranges across breakdown groups', () => { + const events = [ + createEvent('evt1', 'screen_view'), + createEvent('evt2', 'session_start'), + ]; + const series = [ + createSeries(['iOS'], events[0]!, [ + { date: '2025-01-01', count: 100 }, + { date: '2025-01-02', count: 200 }, + ]), + createSeries(['iOS'], events[1]!, [ + { date: '2025-01-01', count: 50 }, + { date: '2025-01-02', count: 100 }, + ]), + createSeries(['Android'], events[0]!, [ + { date: '2025-01-01', count: 300 }, + // Missing 2025-01-02 + ]), + createSeries(['Android'], events[1]!, [ + { date: '2025-01-01', count: 150 }, + { date: '2025-01-02', count: 200 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/B', events }), + series, + ); + + expect(result).toHaveLength(2); + // iOS group + expect(result[0]?.name).toEqual(['iOS']); + expect(result[0]?.data[0]?.count).toBe(2); // 100/50 + expect(result[0]?.data[1]?.count).toBe(2); // 200/100 + // Android group + expect(result[1]?.name).toEqual(['Android']); + expect(result[1]?.data[0]?.count).toBe(2); // 300/150 + expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A) + }); + }); + + describe('complex formulas', () => { + it('should handle complex expressions', () => { + const events = [ + createEvent('evt1', 'event1'), + createEvent('evt2', 'event2'), + createEvent('evt3', 'event3'), + ]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + ]), + createSeries(['event2'], events[1]!, [ + { date: '2025-01-01', count: 20 }, + ]), + createSeries(['event3'], events[2]!, [ + { date: '2025-01-01', count: 30 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: '(A+B)*C', events }), + series, + ); + + // (10+20)*30 = 900 + expect(result[0]?.data[0]?.count).toBe(900); + }); + + it('should handle percentage calculations', () => { + const events = [ + createEvent('evt1', 'screen_view'), + createEvent('evt2', 'session_start'), + ]; + const series = [ + createSeries(['screen_view'], events[0]!, [ + { date: '2025-01-01', count: 75 }, + ]), + createSeries(['session_start'], events[1]!, [ + { date: '2025-01-01', count: 100 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: '(A/B)*100', events }), + series, + ); + + // (75/100)*100 = 75 + expect(result[0]?.data[0]?.count).toBe(75); + }); + }); + + describe('error handling', () => { + it('should handle invalid formulas gracefully', () => { + const events = [createEvent('evt1', 'event1')]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'invalid formula', events }), + series, + ); + + // Should return 0 for invalid formulas + expect(result[0]?.data[0]?.count).toBe(0); + }); + + it('should handle division by zero', () => { + const events = [ + createEvent('evt1', 'event1'), + createEvent('evt2', 'event2'), + ]; + const series = [ + createSeries(['event1'], events[0]!, [ + { date: '2025-01-01', count: 10 }, + ]), + createSeries(['event2'], events[1]!, [ + { date: '2025-01-01', count: 0 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/B', events }), + series, + ); + + // Division by zero should result in 0 (Infinity -> 0) + expect(result[0]?.data[0]?.count).toBe(0); + }); + }); + + describe('real-world scenario: article hit ratio', () => { + it('should calculate hit ratio per article path', () => { + const events = [ + createEvent('evt1', 'screen_view'), + createEvent('evt2', 'article_card_seen'), + ]; + const series = [ + // Article 1 + createSeries(['/articles/1'], events[0]!, [ + { date: '2025-01-01', count: 1000 }, + ]), + createSeries(['/articles/1'], events[1]!, [ + { date: '2025-01-01', count: 100 }, + ]), + // Article 2 + createSeries(['/articles/2'], events[0]!, [ + { date: '2025-01-01', count: 500 }, + ]), + createSeries(['/articles/2'], events[1]!, [ + { date: '2025-01-01', count: 200 }, + ]), + ]; + + const result = withFormula( + createChartInput({ formula: 'A/B', events }), + series, + ); + + expect(result).toHaveLength(2); + // Article 1: 1000/100 = 10 + expect(result[0]?.name).toEqual(['/articles/1']); + expect(result[0]?.data[0]?.count).toBe(10); + // Article 2: 500/200 = 2.5 + expect(result[1]?.name).toEqual(['/articles/2']); + expect(result[1]?.data[0]?.count).toBe(2.5); + }); + }); +}); diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index 788f6e13..40cf884f 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -1,5 +1,5 @@ import * as mathjs from 'mathjs'; -import { last, pluck, reverse, uniq } from 'ramda'; +import { last, reverse } from 'ramda'; import sqlstring from 'sqlstring'; import type { ISerieDataItem } from '@openpanel/common'; @@ -31,7 +31,6 @@ import type { IChartEvent, IChartInput, IChartInputWithDates, - IChartRange, IGetChartDataInput, } from '@openpanel/validation'; @@ -43,74 +42,129 @@ export function withFormula( return series; } - if (!series) { + if (!series || series.length === 0) { return series; } - if (!series[0]) { - return series; - } - if (!series[0].data) { + if (!series[0]?.data) { return series; } - if (events.length === 1) { - return series.map((serie) => { - if (!serie.event.id) { - return serie; + // Formulas always use alphabet IDs (A, B, C, etc.), not event IDs + // Group series by breakdown values (the name array) + // This allows us to match series from different events that have the same breakdown values + + // Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names + // When there are breakdowns, name arrays contain breakdown values (not event names) + const hasBreakdowns = series.some( + (serie) => + serie.name.length > 0 && + !events.some( + (event) => + serie.name[0] === event.name || serie.name[0] === event.displayName, + ), + ); + + const seriesByBreakdown = new Map(); + + series.forEach((serie) => { + let breakdownKey: string; + + if (hasBreakdowns) { + // With breakdowns: use the entire name array as the breakdown key + // The name array contains breakdown values (e.g., ["iOS"], ["Android"]) + breakdownKey = serie.name.join(':::'); + } else { + // Without breakdowns: group all series together regardless of event name + // This allows formulas to combine multiple events + breakdownKey = ''; + } + + if (!seriesByBreakdown.has(breakdownKey)) { + seriesByBreakdown.set(breakdownKey, []); + } + seriesByBreakdown.get(breakdownKey)!.push(serie); + }); + + // For each breakdown group, apply the formula + const result: typeof series = []; + + for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) { + // Group series by event to ensure we have one series per event + const seriesByEvent = new Map(); + + breakdownSeries.forEach((serie) => { + const eventId = serie.event.id ?? serie.event.name; + // If we already have a series for this event in this breakdown group, skip it + // (shouldn't happen, but just in case) + if (!seriesByEvent.has(eventId)) { + seriesByEvent.set(eventId, serie); + } + }); + + // Get all unique dates across all series in this breakdown group + const allDates = new Set(); + breakdownSeries.forEach((serie) => { + serie.data.forEach((item) => { + allDates.add(item.date); + }); + }); + + // Sort dates chronologically + const sortedDates = Array.from(allDates).sort( + (a, b) => new Date(a).getTime() - new Date(b).getTime(), + ); + + // Apply formula for each date, matching series by event index + const formulaData = sortedDates.map((date) => { + const scope: Record = {}; + + // Build scope using alphabet IDs (A, B, C, etc.) for each event + // This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C") + events.forEach((event, eventIndex) => { + const readableId = alphabetIds[eventIndex]; + if (!readableId) { + throw new Error('no alphabet id for serie in withFormula'); + } + + // Find the series for this event in this breakdown group + const eventId = event.id ?? event.name; + const matchingSerie = seriesByEvent.get(eventId); + + // Find the data point for this date + // If the series doesn't exist or the date is missing, use 0 + const dataPoint = matchingSerie?.data.find((d) => d.date === date); + scope[readableId] = dataPoint?.count ?? 0; + }); + + // Evaluate the formula with the scope + let count: number; + try { + count = mathjs.parse(formula).compile().evaluate(scope) as number; + } catch (error) { + // If formula evaluation fails, return 0 + count = 0; } return { - ...serie, - data: serie.data.map((item) => { - serie.event.id; - const scope = { - [serie.event.id ?? '']: item?.count ?? 0, - }; - const count = mathjs - .parse(formula) - .compile() - .evaluate(scope) as number; - - return { - ...item, - count: - Number.isNaN(count) || !Number.isFinite(count) - ? null - : round(count, 2), - }; - }), + date, + count: + Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2), + total_count: breakdownSeries[0]?.data.find((d) => d.date === date) + ?.total_count, }; }); + + // Use the first series as a template, but replace its data with formula results + // Preserve the breakdown labels (name array) from the original series + const templateSerie = breakdownSeries[0]!; + result.push({ + ...templateSerie, + data: formulaData, + }); } - return [ - { - ...series[0], - data: series[0].data.map((item, dIndex) => { - const scope = series.reduce((acc, item, index) => { - const readableId = alphabetIds[index]; - if (!readableId) { - throw new Error('no alphabet id for serie in withFormula'); - } - - return { - ...acc, - [readableId]: item.data[dIndex]?.count ?? 0, - }; - }, {}); - - const count = mathjs.parse(formula).compile().evaluate(scope) as number; - return { - ...item, - count: - Number.isNaN(count) || !Number.isFinite(count) - ? null - : round(count, 2), - }; - }), - }, - ]; + return result; } function fillFunnel(funnel: { level: number; count: number }[], steps: number) { @@ -314,9 +368,7 @@ export async function getChart(input: IChartInput) { const previousSeries = result[1]; const limit = input.limit || 300; const offset = input.offset || 0; - const includeEventName = - uniq(pluck('name', input.events)).length !== - pluck('name', input.events).length && series.length > 1; + const includeEventAlphaId = input.events.length > 1; const final: FinalChart = { series: series.map((serie, index) => { const eventIndex = input.events.findIndex( @@ -343,7 +395,7 @@ export async function getChart(input: IChartInput) { names: input.breakdowns.length === 0 && serie.event.displayName ? [serie.event.displayName] - : includeEventName + : includeEventAlphaId ? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)] : serie.name, event, diff --git a/packages/trpc/vitest.config.ts b/packages/trpc/vitest.config.ts new file mode 100644 index 00000000..f87a2039 --- /dev/null +++ b/packages/trpc/vitest.config.ts @@ -0,0 +1,3 @@ +import { getSharedVitestConfig } from '../../vitest.shared'; + +export default getSharedVitestConfig({ __dirname });