fix: better formula support

packages/trpc/package.json

@@ -4,6 +4,7 @@
   "type": "module",
   "main": "index.ts",
   "scripts": {
+    "test": "vitest",
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {

packages/trpc/src/routers/chart.helpers.test.ts

@@ -0,0 +1,544 @@
+import type { IChartEvent, IChartInput } from '@openpanel/validation';
+import { describe, expect, it } from 'vitest';
+import { withFormula } from './chart.helpers';
+
+// Helper to create a test event
+function createEvent(
+  id: string,
+  name: string,
+  displayName?: string,
+): IChartEvent {
+  return {
+    id,
+    name,
+    displayName: displayName ?? '',
+    segment: 'event',
+    filters: [],
+  };
+}
+
+const createChartInput = (
+  rest: Pick<IChartInput, 'events' | 'formula'>,
+): IChartInput => {
+  return {
+    metric: 'sum',
+    chartType: 'linear',
+    interval: 'day',
+    breakdowns: [],
+    projectId: '1',
+    startDate: '2025-01-01',
+    endDate: '2025-01-01',
+    range: '30d',
+    previous: false,
+    formula: '',
+    ...rest,
+  };
+};
+
+// Helper to create a test series
+function createSeries(
+  name: string[],
+  event: IChartEvent,
+  data: Array<{ date: string; count: number }>,
+) {
+  return {
+    name,
+    event,
+    data: data.map((d) => ({ ...d, total_count: d.count })),
+  };
+}
+
+describe('withFormula', () => {
+  describe('edge cases', () => {
+    it('should return series unchanged when formula is empty', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: '', events }),
+        series,
+      );
+
+      expect(result).toEqual(series);
+    });
+
+    it('should return series unchanged when series is empty', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const result = withFormula(
+        createChartInput({ formula: 'A*100', events }),
+        [],
+      );
+
+      expect(result).toEqual([]);
+    });
+
+    it('should return series unchanged when series has no data', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const series = [{ name: ['event1'], event: events[0]!, data: [] }];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A*100', events }),
+        series,
+      );
+
+      expect(result).toEqual(series);
+    });
+  });
+
+  describe('single event, no breakdown', () => {
+    it('should apply simple multiplication formula', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+          { date: '2025-01-02', count: 20 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A*100', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(1);
+      expect(result[0]?.data).toEqual([
+        { date: '2025-01-01', count: 1000, total_count: 10 },
+        { date: '2025-01-02', count: 2000, total_count: 20 },
+      ]);
+    });
+
+    it('should apply addition formula', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 5 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A+10', events }),
+        series,
+      );
+
+      expect(result[0]?.data[0]?.count).toBe(15);
+    });
+
+    it('should handle division formula', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 100 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/10', events }),
+        series,
+      );
+
+      expect(result[0]?.data[0]?.count).toBe(10);
+    });
+
+    it('should handle NaN and Infinity by returning 0', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 0 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/0', events }),
+        series,
+      );
+
+      expect(result[0]?.data[0]?.count).toBe(0);
+    });
+  });
+
+  describe('single event, with breakdown', () => {
+    it('should apply formula to each breakdown group', () => {
+      const events = [createEvent('evt1', 'screen_view')];
+      const series = [
+        createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]),
+        createSeries(['Android'], events[0]!, [
+          { date: '2025-01-01', count: 20 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A*100', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(2);
+      expect(result[0]?.name).toEqual(['iOS']);
+      expect(result[0]?.data[0]?.count).toBe(1000);
+      expect(result[1]?.name).toEqual(['Android']);
+      expect(result[1]?.data[0]?.count).toBe(2000);
+    });
+
+    it('should handle multiple breakdown values', () => {
+      const events = [createEvent('evt1', 'screen_view')];
+      const series = [
+        createSeries(['iOS', 'US'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+        ]),
+        createSeries(['Android', 'US'], events[0]!, [
+          { date: '2025-01-01', count: 20 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A*2', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(2);
+      expect(result[0]?.name).toEqual(['iOS', 'US']);
+      expect(result[0]?.data[0]?.count).toBe(20);
+      expect(result[1]?.name).toEqual(['Android', 'US']);
+      expect(result[1]?.data[0]?.count).toBe(40);
+    });
+  });
+
+  describe('multiple events, no breakdown', () => {
+    it('should combine two events with division formula', () => {
+      const events = [
+        createEvent('evt1', 'screen_view'),
+        createEvent('evt2', 'session_start'),
+      ];
+      const series = [
+        createSeries(['screen_view'], events[0]!, [
+          { date: '2025-01-01', count: 100 },
+        ]),
+        createSeries(['session_start'], events[1]!, [
+          { date: '2025-01-01', count: 50 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/B', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(1);
+      expect(result[0]?.data[0]?.count).toBe(2);
+    });
+
+    it('should combine two events with addition formula', () => {
+      const events = [
+        createEvent('evt1', 'event1'),
+        createEvent('evt2', 'event2'),
+      ];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+        ]),
+        createSeries(['event2'], events[1]!, [
+          { date: '2025-01-01', count: 20 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A+B', events }),
+        series,
+      );
+
+      expect(result[0]?.data[0]?.count).toBe(30);
+    });
+
+    it('should handle three events', () => {
+      const events = [
+        createEvent('evt1', 'event1'),
+        createEvent('evt2', 'event2'),
+        createEvent('evt3', 'event3'),
+      ];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+        ]),
+        createSeries(['event2'], events[1]!, [
+          { date: '2025-01-01', count: 20 },
+        ]),
+        createSeries(['event3'], events[2]!, [
+          { date: '2025-01-01', count: 30 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A+B+C', events }),
+        series,
+      );
+
+      expect(result[0]?.data[0]?.count).toBe(60);
+    });
+
+    it('should handle missing data points with 0', () => {
+      const events = [
+        createEvent('evt1', 'event1'),
+        createEvent('evt2', 'event2'),
+      ];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+          { date: '2025-01-02', count: 20 },
+        ]),
+        createSeries(['event2'], events[1]!, [
+          { date: '2025-01-01', count: 5 },
+          // Missing 2025-01-02
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A+B', events }),
+        series,
+      );
+
+      expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5
+      expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing)
+    });
+  });
+
+  describe('multiple events, with breakdown', () => {
+    it('should match series by breakdown values and apply formula', () => {
+      const events = [
+        createEvent('evt1', 'screen_view'),
+        createEvent('evt2', 'session_start'),
+      ];
+      const series = [
+        // iOS breakdown
+        createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]),
+        createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]),
+        // Android breakdown
+        createSeries(['Android'], events[0]!, [
+          { date: '2025-01-01', count: 200 },
+        ]),
+        createSeries(['Android'], events[1]!, [
+          { date: '2025-01-01', count: 100 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/B', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(2);
+      // iOS: 100/50 = 2
+      expect(result[0]?.name).toEqual(['iOS']);
+      expect(result[0]?.data[0]?.count).toBe(2);
+      // Android: 200/100 = 2
+      expect(result[1]?.name).toEqual(['Android']);
+      expect(result[1]?.data[0]?.count).toBe(2);
+    });
+
+    it('should handle multiple breakdown values matching', () => {
+      const events = [
+        createEvent('evt1', 'screen_view'),
+        createEvent('evt2', 'session_start'),
+      ];
+      const series = [
+        createSeries(['iOS', 'US'], events[0]!, [
+          { date: '2025-01-01', count: 100 },
+        ]),
+        createSeries(['iOS', 'US'], events[1]!, [
+          { date: '2025-01-01', count: 50 },
+        ]),
+        createSeries(['Android', 'US'], events[0]!, [
+          { date: '2025-01-01', count: 200 },
+        ]),
+        createSeries(['Android', 'US'], events[1]!, [
+          { date: '2025-01-01', count: 100 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/B', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(2);
+      expect(result[0]?.name).toEqual(['iOS', 'US']);
+      expect(result[0]?.data[0]?.count).toBe(2);
+      expect(result[1]?.name).toEqual(['Android', 'US']);
+      expect(result[1]?.data[0]?.count).toBe(2);
+    });
+
+    it('should handle different date ranges across breakdown groups', () => {
+      const events = [
+        createEvent('evt1', 'screen_view'),
+        createEvent('evt2', 'session_start'),
+      ];
+      const series = [
+        createSeries(['iOS'], events[0]!, [
+          { date: '2025-01-01', count: 100 },
+          { date: '2025-01-02', count: 200 },
+        ]),
+        createSeries(['iOS'], events[1]!, [
+          { date: '2025-01-01', count: 50 },
+          { date: '2025-01-02', count: 100 },
+        ]),
+        createSeries(['Android'], events[0]!, [
+          { date: '2025-01-01', count: 300 },
+          // Missing 2025-01-02
+        ]),
+        createSeries(['Android'], events[1]!, [
+          { date: '2025-01-01', count: 150 },
+          { date: '2025-01-02', count: 200 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/B', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(2);
+      // iOS group
+      expect(result[0]?.name).toEqual(['iOS']);
+      expect(result[0]?.data[0]?.count).toBe(2); // 100/50
+      expect(result[0]?.data[1]?.count).toBe(2); // 200/100
+      // Android group
+      expect(result[1]?.name).toEqual(['Android']);
+      expect(result[1]?.data[0]?.count).toBe(2); // 300/150
+      expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A)
+    });
+  });
+
+  describe('complex formulas', () => {
+    it('should handle complex expressions', () => {
+      const events = [
+        createEvent('evt1', 'event1'),
+        createEvent('evt2', 'event2'),
+        createEvent('evt3', 'event3'),
+      ];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+        ]),
+        createSeries(['event2'], events[1]!, [
+          { date: '2025-01-01', count: 20 },
+        ]),
+        createSeries(['event3'], events[2]!, [
+          { date: '2025-01-01', count: 30 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: '(A+B)*C', events }),
+        series,
+      );
+
+      // (10+20)*30 = 900
+      expect(result[0]?.data[0]?.count).toBe(900);
+    });
+
+    it('should handle percentage calculations', () => {
+      const events = [
+        createEvent('evt1', 'screen_view'),
+        createEvent('evt2', 'session_start'),
+      ];
+      const series = [
+        createSeries(['screen_view'], events[0]!, [
+          { date: '2025-01-01', count: 75 },
+        ]),
+        createSeries(['session_start'], events[1]!, [
+          { date: '2025-01-01', count: 100 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: '(A/B)*100', events }),
+        series,
+      );
+
+      // (75/100)*100 = 75
+      expect(result[0]?.data[0]?.count).toBe(75);
+    });
+  });
+
+  describe('error handling', () => {
+    it('should handle invalid formulas gracefully', () => {
+      const events = [createEvent('evt1', 'event1')];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'invalid formula', events }),
+        series,
+      );
+
+      // Should return 0 for invalid formulas
+      expect(result[0]?.data[0]?.count).toBe(0);
+    });
+
+    it('should handle division by zero', () => {
+      const events = [
+        createEvent('evt1', 'event1'),
+        createEvent('evt2', 'event2'),
+      ];
+      const series = [
+        createSeries(['event1'], events[0]!, [
+          { date: '2025-01-01', count: 10 },
+        ]),
+        createSeries(['event2'], events[1]!, [
+          { date: '2025-01-01', count: 0 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/B', events }),
+        series,
+      );
+
+      // Division by zero should result in 0 (Infinity -> 0)
+      expect(result[0]?.data[0]?.count).toBe(0);
+    });
+  });
+
+  describe('real-world scenario: article hit ratio', () => {
+    it('should calculate hit ratio per article path', () => {
+      const events = [
+        createEvent('evt1', 'screen_view'),
+        createEvent('evt2', 'article_card_seen'),
+      ];
+      const series = [
+        // Article 1
+        createSeries(['/articles/1'], events[0]!, [
+          { date: '2025-01-01', count: 1000 },
+        ]),
+        createSeries(['/articles/1'], events[1]!, [
+          { date: '2025-01-01', count: 100 },
+        ]),
+        // Article 2
+        createSeries(['/articles/2'], events[0]!, [
+          { date: '2025-01-01', count: 500 },
+        ]),
+        createSeries(['/articles/2'], events[1]!, [
+          { date: '2025-01-01', count: 200 },
+        ]),
+      ];
+
+      const result = withFormula(
+        createChartInput({ formula: 'A/B', events }),
+        series,
+      );
+
+      expect(result).toHaveLength(2);
+      // Article 1: 1000/100 = 10
+      expect(result[0]?.name).toEqual(['/articles/1']);
+      expect(result[0]?.data[0]?.count).toBe(10);
+      // Article 2: 500/200 = 2.5
+      expect(result[1]?.name).toEqual(['/articles/2']);
+      expect(result[1]?.data[0]?.count).toBe(2.5);
+    });
+  });
+});

packages/trpc/src/routers/chart.helpers.ts

@@ -1,5 +1,5 @@
 import * as mathjs from 'mathjs';
-import { last, pluck, reverse, uniq } from 'ramda';
+import { last, reverse } from 'ramda';
 import sqlstring from 'sqlstring';
 
 import type { ISerieDataItem } from '@openpanel/common';
@@ -31,7 +31,6 @@ import type {
   IChartEvent,
   IChartInput,
   IChartInputWithDates,
-  IChartRange,
   IGetChartDataInput,
 } from '@openpanel/validation';
 
@@ -43,74 +42,129 @@ export function withFormula(
     return series;
   }
 
-  if (!series) {
+  if (!series || series.length === 0) {
     return series;
   }
 
-  if (!series[0]) {
+  if (!series[0]?.data) {
-    return series;
-  }
-  if (!series[0].data) {
     return series;
   }
 
-  if (events.length === 1) {
+  // Formulas always use alphabet IDs (A, B, C, etc.), not event IDs
-    return series.map((serie) => {
+  // Group series by breakdown values (the name array)
-      if (!serie.event.id) {
+  // This allows us to match series from different events that have the same breakdown values
-        return serie;
+
+  // Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names
+  // When there are breakdowns, name arrays contain breakdown values (not event names)
+  const hasBreakdowns = series.some(
+    (serie) =>
+      serie.name.length > 0 &&
+      !events.some(
+        (event) =>
+          serie.name[0] === event.name || serie.name[0] === event.displayName,
+      ),
+  );
+
+  const seriesByBreakdown = new Map<string, typeof series>();
+
+  series.forEach((serie) => {
+    let breakdownKey: string;
+
+    if (hasBreakdowns) {
+      // With breakdowns: use the entire name array as the breakdown key
+      // The name array contains breakdown values (e.g., ["iOS"], ["Android"])
+      breakdownKey = serie.name.join(':::');
+    } else {
+      // Without breakdowns: group all series together regardless of event name
+      // This allows formulas to combine multiple events
+      breakdownKey = '';
     }
 
-      return {
+    if (!seriesByBreakdown.has(breakdownKey)) {
-        ...serie,
+      seriesByBreakdown.set(breakdownKey, []);
-        data: serie.data.map((item) => {
+    }
-          serie.event.id;
+    seriesByBreakdown.get(breakdownKey)!.push(serie);
-          const scope = {
-            [serie.event.id ?? '']: item?.count ?? 0,
-          };
-          const count = mathjs
-            .parse(formula)
-            .compile()
-            .evaluate(scope) as number;
-
-          return {
-            ...item,
-            count:
-              Number.isNaN(count) || !Number.isFinite(count)
-                ? null
-                : round(count, 2),
-          };
-        }),
-      };
   });
-  }
-  return [
-    {
-      ...series[0],
-      data: series[0].data.map((item, dIndex) => {
-        const scope = series.reduce((acc, item, index) => {
-          const readableId = alphabetIds[index];
 
+  // For each breakdown group, apply the formula
+  const result: typeof series = [];
+
+  for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) {
+    // Group series by event to ensure we have one series per event
+    const seriesByEvent = new Map<string, (typeof series)[number]>();
+
+    breakdownSeries.forEach((serie) => {
+      const eventId = serie.event.id ?? serie.event.name;
+      // If we already have a series for this event in this breakdown group, skip it
+      // (shouldn't happen, but just in case)
+      if (!seriesByEvent.has(eventId)) {
+        seriesByEvent.set(eventId, serie);
+      }
+    });
+
+    // Get all unique dates across all series in this breakdown group
+    const allDates = new Set<string>();
+    breakdownSeries.forEach((serie) => {
+      serie.data.forEach((item) => {
+        allDates.add(item.date);
+      });
+    });
+
+    // Sort dates chronologically
+    const sortedDates = Array.from(allDates).sort(
+      (a, b) => new Date(a).getTime() - new Date(b).getTime(),
+    );
+
+    // Apply formula for each date, matching series by event index
+    const formulaData = sortedDates.map((date) => {
+      const scope: Record<string, number> = {};
+
+      // Build scope using alphabet IDs (A, B, C, etc.) for each event
+      // This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C")
+      events.forEach((event, eventIndex) => {
+        const readableId = alphabetIds[eventIndex];
         if (!readableId) {
           throw new Error('no alphabet id for serie in withFormula');
         }
 
-          return {
+        // Find the series for this event in this breakdown group
-            ...acc,
+        const eventId = event.id ?? event.name;
-            [readableId]: item.data[dIndex]?.count ?? 0,
+        const matchingSerie = seriesByEvent.get(eventId);
-          };
+
-        }, {});
+        // Find the data point for this date
+        // If the series doesn't exist or the date is missing, use 0
+        const dataPoint = matchingSerie?.data.find((d) => d.date === date);
+        scope[readableId] = dataPoint?.count ?? 0;
+      });
+
+      // Evaluate the formula with the scope
+      let count: number;
+      try {
+        count = mathjs.parse(formula).compile().evaluate(scope) as number;
+      } catch (error) {
+        // If formula evaluation fails, return 0
+        count = 0;
+      }
 
-        const count = mathjs.parse(formula).compile().evaluate(scope) as number;
       return {
-          ...item,
+        date,
         count:
-            Number.isNaN(count) || !Number.isFinite(count)
+          Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2),
-              ? null
+        total_count: breakdownSeries[0]?.data.find((d) => d.date === date)
-              : round(count, 2),
+          ?.total_count,
       };
-      }),
+    });
-    },
+
-  ];
+    // Use the first series as a template, but replace its data with formula results
+    // Preserve the breakdown labels (name array) from the original series
+    const templateSerie = breakdownSeries[0]!;
+    result.push({
+      ...templateSerie,
+      data: formulaData,
+    });
+  }
+
+  return result;
 }
 
 function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
@@ -314,9 +368,7 @@ export async function getChart(input: IChartInput) {
   const previousSeries = result[1];
   const limit = input.limit || 300;
   const offset = input.offset || 0;
-  const includeEventName =
+  const includeEventAlphaId = input.events.length > 1;
-    uniq(pluck('name', input.events)).length !==
-      pluck('name', input.events).length && series.length > 1;
   const final: FinalChart = {
     series: series.map((serie, index) => {
       const eventIndex = input.events.findIndex(
@@ -343,7 +395,7 @@ export async function getChart(input: IChartInput) {
         names:
           input.breakdowns.length === 0 && serie.event.displayName
             ? [serie.event.displayName]
-            : includeEventName
+            : includeEventAlphaId
               ? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)]
               : serie.name,
         event,

packages/trpc/vitest.config.ts

@@ -0,0 +1,3 @@
+import { getSharedVitestConfig } from '../../vitest.shared';
+
+export default getSharedVitestConfig({ __dirname });