fix: better formula support

This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-12 22:41:04 +01:00
parent 84fd5ce22f
commit c1801adaa2
4 changed files with 661 additions and 61 deletions

View File

@@ -4,6 +4,7 @@
"type": "module",
"main": "index.ts",
"scripts": {
"test": "vitest",
"typecheck": "tsc --noEmit"
},
"dependencies": {

View File

@@ -0,0 +1,544 @@
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { describe, expect, it } from 'vitest';
import { withFormula } from './chart.helpers';
// Helper to create a test event
function createEvent(
id: string,
name: string,
displayName?: string,
): IChartEvent {
return {
id,
name,
displayName: displayName ?? '',
segment: 'event',
filters: [],
};
}
const createChartInput = (
rest: Pick<IChartInput, 'events' | 'formula'>,
): IChartInput => {
return {
metric: 'sum',
chartType: 'linear',
interval: 'day',
breakdowns: [],
projectId: '1',
startDate: '2025-01-01',
endDate: '2025-01-01',
range: '30d',
previous: false,
formula: '',
...rest,
};
};
// Helper to create a test series
function createSeries(
name: string[],
event: IChartEvent,
data: Array<{ date: string; count: number }>,
) {
return {
name,
event,
data: data.map((d) => ({ ...d, total_count: d.count })),
};
}
describe('withFormula', () => {
describe('edge cases', () => {
it('should return series unchanged when formula is empty', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
];
const result = withFormula(
createChartInput({ formula: '', events }),
series,
);
expect(result).toEqual(series);
});
it('should return series unchanged when series is empty', () => {
const events = [createEvent('evt1', 'event1')];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
[],
);
expect(result).toEqual([]);
});
it('should return series unchanged when series has no data', () => {
const events = [createEvent('evt1', 'event1')];
const series = [{ name: ['event1'], event: events[0]!, data: [] }];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toEqual(series);
});
});
describe('single event, no breakdown', () => {
it('should apply simple multiplication formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
{ date: '2025-01-02', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toHaveLength(1);
expect(result[0]?.data).toEqual([
{ date: '2025-01-01', count: 1000, total_count: 10 },
{ date: '2025-01-02', count: 2000, total_count: 20 },
]);
});
it('should apply addition formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 5 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+10', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(15);
});
it('should handle division formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/10', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(10);
});
it('should handle NaN and Infinity by returning 0', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 0 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/0', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(0);
});
});
describe('single event, with breakdown', () => {
it('should apply formula to each breakdown group', () => {
const events = [createEvent('evt1', 'screen_view')];
const series = [
createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]),
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(1000);
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2000);
});
it('should handle multiple breakdown values', () => {
const events = [createEvent('evt1', 'screen_view')];
const series = [
createSeries(['iOS', 'US'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['Android', 'US'], events[0]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*2', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS', 'US']);
expect(result[0]?.data[0]?.count).toBe(20);
expect(result[1]?.name).toEqual(['Android', 'US']);
expect(result[1]?.data[0]?.count).toBe(40);
});
});
describe('multiple events, no breakdown', () => {
it('should combine two events with division formula', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['screen_view'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
createSeries(['session_start'], events[1]!, [
{ date: '2025-01-01', count: 50 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(1);
expect(result[0]?.data[0]?.count).toBe(2);
});
it('should combine two events with addition formula', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(30);
});
it('should handle three events', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
createEvent('evt3', 'event3'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
createSeries(['event3'], events[2]!, [
{ date: '2025-01-01', count: 30 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B+C', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(60);
});
it('should handle missing data points with 0', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
{ date: '2025-01-02', count: 20 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 5 },
// Missing 2025-01-02
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5
expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing)
});
});
describe('multiple events, with breakdown', () => {
it('should match series by breakdown values and apply formula', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
// iOS breakdown
createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]),
createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]),
// Android breakdown
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 200 },
]),
createSeries(['Android'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// iOS: 100/50 = 2
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(2);
// Android: 200/100 = 2
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2);
});
it('should handle multiple breakdown values matching', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['iOS', 'US'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
createSeries(['iOS', 'US'], events[1]!, [
{ date: '2025-01-01', count: 50 },
]),
createSeries(['Android', 'US'], events[0]!, [
{ date: '2025-01-01', count: 200 },
]),
createSeries(['Android', 'US'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS', 'US']);
expect(result[0]?.data[0]?.count).toBe(2);
expect(result[1]?.name).toEqual(['Android', 'US']);
expect(result[1]?.data[0]?.count).toBe(2);
});
it('should handle different date ranges across breakdown groups', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['iOS'], events[0]!, [
{ date: '2025-01-01', count: 100 },
{ date: '2025-01-02', count: 200 },
]),
createSeries(['iOS'], events[1]!, [
{ date: '2025-01-01', count: 50 },
{ date: '2025-01-02', count: 100 },
]),
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 300 },
// Missing 2025-01-02
]),
createSeries(['Android'], events[1]!, [
{ date: '2025-01-01', count: 150 },
{ date: '2025-01-02', count: 200 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// iOS group
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(2); // 100/50
expect(result[0]?.data[1]?.count).toBe(2); // 200/100
// Android group
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2); // 300/150
expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A)
});
});
describe('complex formulas', () => {
it('should handle complex expressions', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
createEvent('evt3', 'event3'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
createSeries(['event3'], events[2]!, [
{ date: '2025-01-01', count: 30 },
]),
];
const result = withFormula(
createChartInput({ formula: '(A+B)*C', events }),
series,
);
// (10+20)*30 = 900
expect(result[0]?.data[0]?.count).toBe(900);
});
it('should handle percentage calculations', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['screen_view'], events[0]!, [
{ date: '2025-01-01', count: 75 },
]),
createSeries(['session_start'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: '(A/B)*100', events }),
series,
);
// (75/100)*100 = 75
expect(result[0]?.data[0]?.count).toBe(75);
});
});
describe('error handling', () => {
it('should handle invalid formulas gracefully', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
];
const result = withFormula(
createChartInput({ formula: 'invalid formula', events }),
series,
);
// Should return 0 for invalid formulas
expect(result[0]?.data[0]?.count).toBe(0);
});
it('should handle division by zero', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 0 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
// Division by zero should result in 0 (Infinity -> 0)
expect(result[0]?.data[0]?.count).toBe(0);
});
});
describe('real-world scenario: article hit ratio', () => {
it('should calculate hit ratio per article path', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'article_card_seen'),
];
const series = [
// Article 1
createSeries(['/articles/1'], events[0]!, [
{ date: '2025-01-01', count: 1000 },
]),
createSeries(['/articles/1'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
// Article 2
createSeries(['/articles/2'], events[0]!, [
{ date: '2025-01-01', count: 500 },
]),
createSeries(['/articles/2'], events[1]!, [
{ date: '2025-01-01', count: 200 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// Article 1: 1000/100 = 10
expect(result[0]?.name).toEqual(['/articles/1']);
expect(result[0]?.data[0]?.count).toBe(10);
// Article 2: 500/200 = 2.5
expect(result[1]?.name).toEqual(['/articles/2']);
expect(result[1]?.data[0]?.count).toBe(2.5);
});
});
});

View File

@@ -1,5 +1,5 @@
import * as mathjs from 'mathjs';
import { last, pluck, reverse, uniq } from 'ramda';
import { last, reverse } from 'ramda';
import sqlstring from 'sqlstring';
import type { ISerieDataItem } from '@openpanel/common';
@@ -31,7 +31,6 @@ import type {
IChartEvent,
IChartInput,
IChartInputWithDates,
IChartRange,
IGetChartDataInput,
} from '@openpanel/validation';
@@ -43,74 +42,129 @@ export function withFormula(
return series;
}
if (!series) {
if (!series || series.length === 0) {
return series;
}
if (!series[0]) {
return series;
}
if (!series[0].data) {
if (!series[0]?.data) {
return series;
}
if (events.length === 1) {
return series.map((serie) => {
if (!serie.event.id) {
return serie;
// Formulas always use alphabet IDs (A, B, C, etc.), not event IDs
// Group series by breakdown values (the name array)
// This allows us to match series from different events that have the same breakdown values
// Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names
// When there are breakdowns, name arrays contain breakdown values (not event names)
const hasBreakdowns = series.some(
(serie) =>
serie.name.length > 0 &&
!events.some(
(event) =>
serie.name[0] === event.name || serie.name[0] === event.displayName,
),
);
const seriesByBreakdown = new Map<string, typeof series>();
series.forEach((serie) => {
let breakdownKey: string;
if (hasBreakdowns) {
// With breakdowns: use the entire name array as the breakdown key
// The name array contains breakdown values (e.g., ["iOS"], ["Android"])
breakdownKey = serie.name.join(':::');
} else {
// Without breakdowns: group all series together regardless of event name
// This allows formulas to combine multiple events
breakdownKey = '';
}
if (!seriesByBreakdown.has(breakdownKey)) {
seriesByBreakdown.set(breakdownKey, []);
}
seriesByBreakdown.get(breakdownKey)!.push(serie);
});
// For each breakdown group, apply the formula
const result: typeof series = [];
for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) {
// Group series by event to ensure we have one series per event
const seriesByEvent = new Map<string, (typeof series)[number]>();
breakdownSeries.forEach((serie) => {
const eventId = serie.event.id ?? serie.event.name;
// If we already have a series for this event in this breakdown group, skip it
// (shouldn't happen, but just in case)
if (!seriesByEvent.has(eventId)) {
seriesByEvent.set(eventId, serie);
}
});
// Get all unique dates across all series in this breakdown group
const allDates = new Set<string>();
breakdownSeries.forEach((serie) => {
serie.data.forEach((item) => {
allDates.add(item.date);
});
});
// Sort dates chronologically
const sortedDates = Array.from(allDates).sort(
(a, b) => new Date(a).getTime() - new Date(b).getTime(),
);
// Apply formula for each date, matching series by event index
const formulaData = sortedDates.map((date) => {
const scope: Record<string, number> = {};
// Build scope using alphabet IDs (A, B, C, etc.) for each event
// This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C")
events.forEach((event, eventIndex) => {
const readableId = alphabetIds[eventIndex];
if (!readableId) {
throw new Error('no alphabet id for serie in withFormula');
}
// Find the series for this event in this breakdown group
const eventId = event.id ?? event.name;
const matchingSerie = seriesByEvent.get(eventId);
// Find the data point for this date
// If the series doesn't exist or the date is missing, use 0
const dataPoint = matchingSerie?.data.find((d) => d.date === date);
scope[readableId] = dataPoint?.count ?? 0;
});
// Evaluate the formula with the scope
let count: number;
try {
count = mathjs.parse(formula).compile().evaluate(scope) as number;
} catch (error) {
// If formula evaluation fails, return 0
count = 0;
}
return {
...serie,
data: serie.data.map((item) => {
serie.event.id;
const scope = {
[serie.event.id ?? '']: item?.count ?? 0,
};
const count = mathjs
.parse(formula)
.compile()
.evaluate(scope) as number;
return {
...item,
count:
Number.isNaN(count) || !Number.isFinite(count)
? null
: round(count, 2),
};
}),
date,
count:
Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2),
total_count: breakdownSeries[0]?.data.find((d) => d.date === date)
?.total_count,
};
});
// Use the first series as a template, but replace its data with formula results
// Preserve the breakdown labels (name array) from the original series
const templateSerie = breakdownSeries[0]!;
result.push({
...templateSerie,
data: formulaData,
});
}
return [
{
...series[0],
data: series[0].data.map((item, dIndex) => {
const scope = series.reduce((acc, item, index) => {
const readableId = alphabetIds[index];
if (!readableId) {
throw new Error('no alphabet id for serie in withFormula');
}
return {
...acc,
[readableId]: item.data[dIndex]?.count ?? 0,
};
}, {});
const count = mathjs.parse(formula).compile().evaluate(scope) as number;
return {
...item,
count:
Number.isNaN(count) || !Number.isFinite(count)
? null
: round(count, 2),
};
}),
},
];
return result;
}
function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
@@ -314,9 +368,7 @@ export async function getChart(input: IChartInput) {
const previousSeries = result[1];
const limit = input.limit || 300;
const offset = input.offset || 0;
const includeEventName =
uniq(pluck('name', input.events)).length !==
pluck('name', input.events).length && series.length > 1;
const includeEventAlphaId = input.events.length > 1;
const final: FinalChart = {
series: series.map((serie, index) => {
const eventIndex = input.events.findIndex(
@@ -343,7 +395,7 @@ export async function getChart(input: IChartInput) {
names:
input.breakdowns.length === 0 && serie.event.displayName
? [serie.event.displayName]
: includeEventName
: includeEventAlphaId
? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)]
: serie.name,
event,

View File

@@ -0,0 +1,3 @@
import { getSharedVitestConfig } from '../../vitest.shared';
export default getSharedVitestConfig({ __dirname });