zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: report editor

Browse Source 
commit bfcf271a64c33a60f61f511cec2198d9c8a9c51a
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Wed Nov 26 12:32:40 2025 +0100

    wip

commit 8cd3b89fa3
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 22:33:58 2025 +0100

    funnel

commit 95af86dc44
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 22:23:25 2025 +0100

    wip

commit 727a218e6b
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 10:18:26 2025 +0100

    conversion wip

commit 958ba535d6
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 10:18:20 2025 +0100

    wip

commit 3bbeb927cc
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 09:18:48 2025 +0100

    wip

commit d99335e2f4
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 18:08:10 2025 +0100

    wip

commit 1fa61b1ae9
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 15:50:28 2025 +0100

    ts

commit 548747d826
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 13:17:01 2025 +0100

    fix typecheck events -> series

commit 7b18544085
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 13:06:46 2025 +0100

    fix report table

commit 57697a5a39
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Sat Nov 22 00:05:13 2025 +0100

    wip

commit 06fb6c4f3c
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Fri Nov 21 11:21:17 2025 +0100

    wip

commit dd71fd4e11
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Thu Nov 20 13:56:58 2025 +0100

    formulas
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-26 12:33:41 +01:00
parent 828c8c4f91
commit b421474616
70 changed files with 6867 additions and 1918 deletions
Download Patch File Download Diff File Expand all files Collapse all files

544
packages/trpc/src/routers/chart.helpers.test.ts
View File

@@ -1,544 +0,0 @@
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { describe, expect, it } from 'vitest';
import { withFormula } from './chart.helpers';
// Helper to create a test event
function createEvent(
id: string,
name: string,
displayName?: string,
): IChartEvent {
return {
id,
name,
displayName: displayName ?? '',
segment: 'event',
filters: [],
};
}
const createChartInput = (
rest: Pick<IChartInput, 'events' | 'formula'>,
): IChartInput => {
return {
metric: 'sum',
chartType: 'linear',
interval: 'day',
breakdowns: [],
projectId: '1',
startDate: '2025-01-01',
endDate: '2025-01-01',
range: '30d',
previous: false,
formula: '',
...rest,
};
};
// Helper to create a test series
function createSeries(
name: string[],
event: IChartEvent,
data: Array<{ date: string; count: number }>,
) {
return {
name,
event,
data: data.map((d) => ({ ...d, total_count: d.count })),
};
}
describe('withFormula', () => {
describe('edge cases', () => {
it('should return series unchanged when formula is empty', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
];
const result = withFormula(
createChartInput({ formula: '', events }),
series,
);
expect(result).toEqual(series);
});
it('should return series unchanged when series is empty', () => {
const events = [createEvent('evt1', 'event1')];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
[],
);
expect(result).toEqual([]);
});
it('should return series unchanged when series has no data', () => {
const events = [createEvent('evt1', 'event1')];
const series = [{ name: ['event1'], event: events[0]!, data: [] }];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toEqual(series);
});
});
describe('single event, no breakdown', () => {
it('should apply simple multiplication formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
{ date: '2025-01-02', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toHaveLength(1);
expect(result[0]?.data).toEqual([
{ date: '2025-01-01', count: 1000, total_count: 10 },
{ date: '2025-01-02', count: 2000, total_count: 20 },
]);
});
it('should apply addition formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 5 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+10', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(15);
});
it('should handle division formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/10', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(10);
});
it('should handle NaN and Infinity by returning 0', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 0 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/0', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(0);
});
});
describe('single event, with breakdown', () => {
it('should apply formula to each breakdown group', () => {
const events = [createEvent('evt1', 'screen_view')];
const series = [
createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]),
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(1000);
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2000);
});
it('should handle multiple breakdown values', () => {
const events = [createEvent('evt1', 'screen_view')];
const series = [
createSeries(['iOS', 'US'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['Android', 'US'], events[0]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*2', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS', 'US']);
expect(result[0]?.data[0]?.count).toBe(20);
expect(result[1]?.name).toEqual(['Android', 'US']);
expect(result[1]?.data[0]?.count).toBe(40);
});
});
describe('multiple events, no breakdown', () => {
it('should combine two events with division formula', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['screen_view'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
createSeries(['session_start'], events[1]!, [
{ date: '2025-01-01', count: 50 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(1);
expect(result[0]?.data[0]?.count).toBe(2);
});
it('should combine two events with addition formula', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(30);
});
it('should handle three events', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
createEvent('evt3', 'event3'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
createSeries(['event3'], events[2]!, [
{ date: '2025-01-01', count: 30 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B+C', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(60);
});
it('should handle missing data points with 0', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
{ date: '2025-01-02', count: 20 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 5 },
// Missing 2025-01-02
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5
expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing)
});
});
describe('multiple events, with breakdown', () => {
it('should match series by breakdown values and apply formula', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
// iOS breakdown
createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]),
createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]),
// Android breakdown
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 200 },
]),
createSeries(['Android'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// iOS: 100/50 = 2
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(2);
// Android: 200/100 = 2
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2);
});
it('should handle multiple breakdown values matching', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['iOS', 'US'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
createSeries(['iOS', 'US'], events[1]!, [
{ date: '2025-01-01', count: 50 },
]),
createSeries(['Android', 'US'], events[0]!, [
{ date: '2025-01-01', count: 200 },
]),
createSeries(['Android', 'US'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS', 'US']);
expect(result[0]?.data[0]?.count).toBe(2);
expect(result[1]?.name).toEqual(['Android', 'US']);
expect(result[1]?.data[0]?.count).toBe(2);
});
it('should handle different date ranges across breakdown groups', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['iOS'], events[0]!, [
{ date: '2025-01-01', count: 100 },
{ date: '2025-01-02', count: 200 },
]),
createSeries(['iOS'], events[1]!, [
{ date: '2025-01-01', count: 50 },
{ date: '2025-01-02', count: 100 },
]),
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 300 },
// Missing 2025-01-02
]),
createSeries(['Android'], events[1]!, [
{ date: '2025-01-01', count: 150 },
{ date: '2025-01-02', count: 200 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// iOS group
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(2); // 100/50
expect(result[0]?.data[1]?.count).toBe(2); // 200/100
// Android group
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2); // 300/150
expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A)
});
});
describe('complex formulas', () => {
it('should handle complex expressions', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
createEvent('evt3', 'event3'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
createSeries(['event3'], events[2]!, [
{ date: '2025-01-01', count: 30 },
]),
];
const result = withFormula(
createChartInput({ formula: '(A+B)*C', events }),
series,
);
// (10+20)*30 = 900
expect(result[0]?.data[0]?.count).toBe(900);
});
it('should handle percentage calculations', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['screen_view'], events[0]!, [
{ date: '2025-01-01', count: 75 },
]),
createSeries(['session_start'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: '(A/B)*100', events }),
series,
);
// (75/100)*100 = 75
expect(result[0]?.data[0]?.count).toBe(75);
});
});
describe('error handling', () => {
it('should handle invalid formulas gracefully', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
];
const result = withFormula(
createChartInput({ formula: 'invalid formula', events }),
series,
);
// Should return 0 for invalid formulas
expect(result[0]?.data[0]?.count).toBe(0);
});
it('should handle division by zero', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 0 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
// Division by zero should result in 0 (Infinity -> 0)
expect(result[0]?.data[0]?.count).toBe(0);
});
});
describe('real-world scenario: article hit ratio', () => {
it('should calculate hit ratio per article path', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'article_card_seen'),
];
const series = [
// Article 1
createSeries(['/articles/1'], events[0]!, [
{ date: '2025-01-01', count: 1000 },
]),
createSeries(['/articles/1'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
// Article 2
createSeries(['/articles/2'], events[0]!, [
{ date: '2025-01-01', count: 500 },
]),
createSeries(['/articles/2'], events[1]!, [
{ date: '2025-01-01', count: 200 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// Article 1: 1000/100 = 10
expect(result[0]?.name).toEqual(['/articles/1']);
expect(result[0]?.data[0]?.count).toBe(10);
// Article 2: 500/200 = 2.5
expect(result[1]?.name).toEqual(['/articles/2']);
expect(result[1]?.data[0]?.count).toBe(2.5);
});
});
});

514
packages/trpc/src/routers/chart.helpers.ts
View File

@@ -1,514 +0,0 @@
import * as mathjs from 'mathjs';
import { last, reverse } from 'ramda';
import sqlstring from 'sqlstring';
import type { ISerieDataItem } from '@openpanel/common';
import {
average,
getPreviousMetric,
groupByLabels,
max,
min,
round,
slug,
sum,
} from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import {
TABLE_NAMES,
chQuery,
createSqlBuilder,
formatClickhouseDate,
getChartPrevStartEndDate,
getChartSql,
getChartStartEndDate,
getEventFiltersWhereClause,
getOrganizationSubscriptionChartEndDate,
getSettingsForProject,
} from '@openpanel/db';
import type {
FinalChart,
IChartEvent,
IChartInput,
IChartInputWithDates,
IGetChartDataInput,
} from '@openpanel/validation';
export function withFormula(
{ formula, events }: IChartInput,
series: Awaited<ReturnType<typeof getChartSerie>>,
) {
if (!formula) {
return series;
}
if (!series || series.length === 0) {
return series;
}
if (!series[0]?.data) {
return series;
}
// Formulas always use alphabet IDs (A, B, C, etc.), not event IDs
// Group series by breakdown values (the name array)
// This allows us to match series from different events that have the same breakdown values
// Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names
// When there are breakdowns, name arrays contain breakdown values (not event names)
const hasBreakdowns = series.some(
(serie) =>
serie.name.length > 0 &&
!events.some(
(event) =>
serie.name[0] === event.name || serie.name[0] === event.displayName,
),
);
const seriesByBreakdown = new Map<string, typeof series>();
series.forEach((serie) => {
let breakdownKey: string;
if (hasBreakdowns) {
// With breakdowns: use the entire name array as the breakdown key
// The name array contains breakdown values (e.g., ["iOS"], ["Android"])
breakdownKey = serie.name.join(':::');
} else {
// Without breakdowns: group all series together regardless of event name
// This allows formulas to combine multiple events
breakdownKey = '';
}
if (!seriesByBreakdown.has(breakdownKey)) {
seriesByBreakdown.set(breakdownKey, []);
}
seriesByBreakdown.get(breakdownKey)!.push(serie);
});
// For each breakdown group, apply the formula
const result: typeof series = [];
for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) {
// Group series by event to ensure we have one series per event
const seriesByEvent = new Map<string, (typeof series)[number]>();
breakdownSeries.forEach((serie) => {
const eventId = serie.event.id ?? serie.event.name;
// If we already have a series for this event in this breakdown group, skip it
// (shouldn't happen, but just in case)
if (!seriesByEvent.has(eventId)) {
seriesByEvent.set(eventId, serie);
}
});
// Get all unique dates across all series in this breakdown group
const allDates = new Set<string>();
breakdownSeries.forEach((serie) => {
serie.data.forEach((item) => {
allDates.add(item.date);
});
});
// Sort dates chronologically
const sortedDates = Array.from(allDates).sort(
(a, b) => new Date(a).getTime() - new Date(b).getTime(),
);
// Apply formula for each date, matching series by event index
const formulaData = sortedDates.map((date) => {
const scope: Record<string, number> = {};
// Build scope using alphabet IDs (A, B, C, etc.) for each event
// This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C")
events.forEach((event, eventIndex) => {
const readableId = alphabetIds[eventIndex];
if (!readableId) {
throw new Error('no alphabet id for serie in withFormula');
}
// Find the series for this event in this breakdown group
const eventId = event.id ?? event.name;
const matchingSerie = seriesByEvent.get(eventId);
// Find the data point for this date
// If the series doesn't exist or the date is missing, use 0
const dataPoint = matchingSerie?.data.find((d) => d.date === date);
scope[readableId] = dataPoint?.count ?? 0;
});
// Evaluate the formula with the scope
let count: number;
try {
count = mathjs.parse(formula).compile().evaluate(scope) as number;
} catch (error) {
// If formula evaluation fails, return 0
count = 0;
}
return {
date,
count:
Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2),
total_count: breakdownSeries[0]?.data.find((d) => d.date === date)
?.total_count,
};
});
// Use the first series as a template, but replace its data with formula results
// Preserve the breakdown labels (name array) from the original series
const templateSerie = breakdownSeries[0]!;
result.push({
...templateSerie,
data: formulaData,
});
}
return result;
}
function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
const filled = Array.from({ length: steps }, (_, index) => {
const level = index + 1;
const matchingResult = funnel.find((res) => res.level === level);
return {
level,
count: matchingResult ? matchingResult.count : 0,
};
});
// Accumulate counts from top to bottom of the funnel
for (let i = filled.length - 1; i >= 0; i--) {
const step = filled[i];
const prevStep = filled[i + 1];
// If there's a previous step, add the count to the current step
if (step && prevStep) {
step.count += prevStep.count;
}
}
return filled.reverse();
}
export async function getFunnelData({
projectId,
startDate,
endDate,
...payload
}: IChartInput) {
const funnelWindow = (payload.funnelWindow || 24) * 3600;
const funnelGroup =
payload.funnelGroup === 'profile_id'
? [`COALESCE(nullIf(s.profile_id, ''), e.profile_id)`, 'profile_id']
: ['session_id', 'session_id'];
if (!startDate || !endDate) {
throw new Error('startDate and endDate are required');
}
if (payload.events.length === 0) {
return {
totalSessions: 0,
steps: [],
};
}
const funnels = payload.events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
const commonWhere = `project_id = ${sqlstring.escape(projectId)} AND
created_at >= '${formatClickhouseDate(startDate)}' AND
created_at <= '${formatClickhouseDate(endDate)}'`;
const innerSql = `SELECT
${funnelGroup[0]} AS ${funnelGroup[1]},
windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
FROM ${TABLE_NAMES.events} e
${funnelGroup[0] === 'session_id' ? '' : `LEFT JOIN (SELECT profile_id, id FROM sessions WHERE ${commonWhere}) AS s ON s.id = e.session_id`}
WHERE
${commonWhere} AND
name IN (${payload.events.map((event) => sqlstring.escape(event.name)).join(', ')})
GROUP BY ${funnelGroup[0]}`;
const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;
const funnel = await chQuery<{ level: number; count: number }>(sql);
const maxLevel = payload.events.length;
const filledFunnelRes = fillFunnel(funnel, maxLevel);
const totalSessions = last(filledFunnelRes)?.count ?? 0;
const steps = reverse(filledFunnelRes).reduce(
(acc, item, index, list) => {
const prev = list[index - 1] ?? { count: totalSessions };
const event = payload.events[item.level - 1]!;
return [
...acc,
{
event: {
...event,
displayName: event.displayName ?? event.name,
},
count: item.count,
percent: (item.count / totalSessions) * 100,
dropoffCount: prev.count - item.count,
dropoffPercent: 100 - (item.count / prev.count) * 100,
previousCount: prev.count,
},
];
},
[] as {
event: IChartEvent & { displayName: string };
count: number;
percent: number;
dropoffCount: number;
dropoffPercent: number;
previousCount: number;
}[],
);
return {
totalSessions,
steps,
};
}
export async function getChartSerie(
payload: IGetChartDataInput,
timezone: string,
) {
let result = await chQuery<ISerieDataItem>(
getChartSql({ ...payload, timezone }),
{
session_timezone: timezone,
},
);
if (result.length === 0 && payload.breakdowns.length > 0) {
result = await chQuery<ISerieDataItem>(
getChartSql({
...payload,
breakdowns: [],
timezone,
}),
{
session_timezone: timezone,
},
);
}
return groupByLabels(result).map((serie) => {
return {
...serie,
event: payload.event,
};
});
}
export type IGetChartSerie = Awaited<ReturnType<typeof getChartSeries>>[number];
export async function getChartSeries(
input: IChartInputWithDates,
timezone: string,
) {
const series = (
await Promise.all(
input.events.map(async (event) =>
getChartSerie(
{
...input,
event,
},
timezone,
),
),
)
).flat();
try {
return withFormula(input, series);
} catch (e) {
return series;
}
}
export async function getChart(input: IChartInput) {
const { timezone } = await getSettingsForProject(input.projectId);
const currentPeriod = getChartStartEndDate(input, timezone);
const previousPeriod = getChartPrevStartEndDate(currentPeriod);
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
currentPeriod.endDate,
);
if (endDate) {
currentPeriod.endDate = endDate;
}
const promises = [getChartSeries({ ...input, ...currentPeriod }, timezone)];
if (input.previous) {
promises.push(
getChartSeries(
{
...input,
...previousPeriod,
},
timezone,
),
);
}
const getSerieId = (serie: IGetChartSerie) =>
[slug(serie.name.join('-')), serie.event.id].filter(Boolean).join('-');
const result = await Promise.all(promises);
const series = result[0]!;
const previousSeries = result[1];
const limit = input.limit || 300;
const offset = input.offset || 0;
const includeEventAlphaId = input.events.length > 1;
const final: FinalChart = {
series: series.map((serie, index) => {
const eventIndex = input.events.findIndex(
(event) => event.id === serie.event.id,
);
const alphaId = alphabetIds[eventIndex];
const previousSerie = previousSeries?.find(
(prevSerie) => getSerieId(prevSerie) === getSerieId(serie),
);
const metrics = {
sum: sum(serie.data.map((item) => item.count)),
average: round(average(serie.data.map((item) => item.count)), 2),
min: min(serie.data.map((item) => item.count)),
max: max(serie.data.map((item) => item.count)),
count: serie.data[0]?.total_count, // We can grab any since all are the same
};
const event = {
id: serie.event.id,
name: serie.event.displayName || serie.event.name,
};
return {
id: getSerieId(serie),
names:
input.breakdowns.length === 0 && serie.event.displayName
? [serie.event.displayName]
: includeEventAlphaId
? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)]
: serie.name,
event,
metrics: {
...metrics,
...(input.previous
? {
previous: {
sum: getPreviousMetric(
metrics.sum,
previousSerie
? sum(previousSerie?.data.map((item) => item.count))
: null,
),
average: getPreviousMetric(
metrics.average,
previousSerie
? round(
average(
previousSerie?.data.map((item) => item.count),
),
2,
)
: null,
),
min: getPreviousMetric(
metrics.sum,
previousSerie
? min(previousSerie?.data.map((item) => item.count))
: null,
),
max: getPreviousMetric(
metrics.sum,
previousSerie
? max(previousSerie?.data.map((item) => item.count))
: null,
),
count: getPreviousMetric(
metrics.count ?? 0,
previousSerie?.data[0]?.total_count ?? null,
),
},
}
: {}),
},
data: serie.data.map((item, index) => ({
date: item.date,
count: item.count ?? 0,
previous: previousSerie?.data[index]
? getPreviousMetric(
item.count ?? 0,
previousSerie?.data[index]?.count ?? null,
)
: undefined,
})),
};
}),
metrics: {
sum: 0,
average: 0,
min: 0,
max: 0,
count: undefined,
},
};
// Sort by sum
final.series = final.series
.sort((a, b) => {
if (input.chartType === 'linear') {
const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
return sumB - sumA;
}
return (b.metrics[input.metric] ?? 0) - (a.metrics[input.metric] ?? 0);
})
.slice(offset, limit ? offset + limit : series.length);
final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
final.metrics.average = round(
average(final.series.map((item) => item.metrics.average)),
2,
);
final.metrics.min = min(final.series.map((item) => item.metrics.min));
final.metrics.max = max(final.series.map((item) => item.metrics.max));
if (input.previous) {
final.metrics.previous = {
sum: getPreviousMetric(
final.metrics.sum,
sum(final.series.map((item) => item.metrics.previous?.sum?.value ?? 0)),
),
average: getPreviousMetric(
final.metrics.average,
round(
average(
final.series.map(
(item) => item.metrics.previous?.average?.value ?? 0,
),
),
2,
),
),
min: getPreviousMetric(
final.metrics.min,
min(final.series.map((item) => item.metrics.previous?.min?.value ?? 0)),
),
max: getPreviousMetric(
final.metrics.max,
max(final.series.map((item) => item.metrics.previous?.max?.value ?? 0)),
),
count: undefined,
};
}
return final;
}

208
packages/trpc/src/routers/chart.ts
View File

@@ -10,22 +10,32 @@ import {
chQuery,
clix,
conversionService,
createSqlBuilder,
db,
formatClickhouseDate,
funnelService,
getChartPrevStartEndDate,
getChartStartEndDate,
getEventFiltersWhereClause,
getEventMetasCached,
getProfilesCached,
getSelectPropertyKey,
getSettingsForProject,
onlyReportEvents,
} from '@openpanel/db';
import {
type IChartEvent,
zChartEvent,
zChartEventFilter,
zChartInput,
zChartSeries,
zCriteria,
zRange,
zTimeInterval,
} from '@openpanel/validation';
import { round } from '@openpanel/common';
import { ChartEngine } from '@openpanel/db';
import {
differenceInDays,
differenceInMonths,
@@ -40,7 +50,6 @@ import {
protectedProcedure,
publicProcedure,
} from '../trpc';
import { getChart } from './chart.helpers';
function utc(date: string | Date) {
if (typeof date === 'string') {
@@ -402,7 +411,8 @@ export const chartRouter = createTRPCRouter({
}
}
return getChart(input);
// Use new chart engine
return ChartEngine.execute(input);
}),
cohort: protectedProcedure
.input(
@@ -532,6 +542,200 @@ export const chartRouter = createTRPCRouter({
return processCohortData(cohortData, diffInterval);
}),
getProfiles: protectedProcedure
.input(
z.object({
projectId: z.string(),
date: z.string().describe('The date for the data point (ISO string)'),
interval: zTimeInterval.default('day'),
series: zChartSeries,
breakdowns: z.record(z.string(), z.string()).optional(),
}),
)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const { projectId, date, series } = input;
const limit = 100;
const serie = series[0];
if (!serie) {
throw new Error('Series not found');
}
if (serie.type !== 'event') {
throw new Error('Series must be an event');
}
// Build the date range for the specific interval bucket
const dateObj = new Date(date);
// Build query to get unique profile_ids for this time bucket
const { sb, getSql } = createSqlBuilder();
sb.select.profile_id = 'DISTINCT profile_id';
sb.where = getEventFiltersWhereClause(serie.filters);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
sb.where.dateRange = `${clix.toStartOf('created_at', input.interval)} = ${clix.toDate(sqlstring.escape(formatClickhouseDate(dateObj)), input.interval)}`;
if (serie.name !== '*') {
sb.where.eventName = `name = ${sqlstring.escape(serie.name)}`;
}
console.log('> breakdowns', input.breakdowns);
if (input.breakdowns) {
Object.entries(input.breakdowns).forEach(([key, value]) => {
sb.where[`breakdown_${key}`] = `${key} = ${sqlstring.escape(value)}`;
});
}
// // Handle breakdowns if provided
// const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
// breakdown.name.startsWith('profile.'),
// );
// const anyFilterOnProfile = [...event.filters, ...filters].some((filter) =>
// filter.name.startsWith('profile.'),
// );
// if (anyFilterOnProfile || anyBreakdownOnProfile) {
// sb.joins.profiles = `LEFT ANY JOIN (SELECT
// id as "profile.id",
// email as "profile.email",
// first_name as "profile.first_name",
// last_name as "profile.last_name",
// properties as "profile.properties"
// FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
// }
// Apply breakdown filters if provided
// breakdowns.forEach((breakdown) => {
// // This is simplified - in reality we'd need to match the breakdown value
// // For now, we'll just get all profiles for the time bucket
// });
// Get unique profile IDs
const profileIds = await chQuery<{ profile_id: string }>(getSql());
if (profileIds.length === 0) {
return [];
}
// Fetch profile details
const ids = profileIds.map((p) => p.profile_id).filter(Boolean);
const profiles = await getProfilesCached(ids, projectId);
return profiles;
}),
getFunnelProfiles: protectedProcedure
.input(
z.object({
projectId: z.string(),
startDate: z.string().nullish(),
endDate: z.string().nullish(),
series: zChartSeries,
stepIndex: z.number().describe('0-based index of the funnel step'),
showDropoffs: z
.boolean()
.optional()
.default(false)
.describe(
'If true, show users who dropped off at this step. If false, show users who completed at least this step.',
),
funnelWindow: z.number().optional(),
funnelGroup: z.string().optional(),
breakdowns: z.array(z.object({ name: z.string() })).optional(),
range: zRange,
}),
)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const {
projectId,
series,
stepIndex,
showDropoffs = false,
funnelWindow,
funnelGroup,
breakdowns = [],
} = input;
const { startDate, endDate } = getChartStartEndDate(input, timezone);
// stepIndex is 0-based, but level is 1-based, so we need level >= stepIndex + 1
const targetLevel = stepIndex + 1;
const eventSeries = onlyReportEvents(series);
if (eventSeries.length === 0) {
throw new Error('At least one event series is required');
}
const funnelWindowSeconds = (funnelWindow || 24) * 3600;
const funnelWindowMilliseconds = funnelWindowSeconds * 1000;
// Use funnel service methods
const group = funnelService.getFunnelGroup(funnelGroup);
// Create sessions CTE if needed
const sessionsCte =
group[0] !== 'session_id'
? funnelService.buildSessionsCte({
projectId,
startDate,
endDate,
timezone,
})
: null;
// Create funnel CTE using funnel service
const funnelCte = funnelService.buildFunnelCte({
projectId,
startDate,
endDate,
eventSeries: eventSeries as IChartEvent[],
funnelWindowMilliseconds,
group,
timezone,
additionalSelects: ['profile_id'],
additionalGroupBy: ['profile_id'],
});
// Build main query
const query = clix(ch, timezone);
if (sessionsCte) {
funnelCte.leftJoin('sessions s', 's.sid = events.session_id');
query.with('sessions', sessionsCte);
}
query.with('funnel', funnelCte);
// Get distinct profile IDs
query
.select(['DISTINCT profile_id'])
.from('funnel')
.where('level', '!=', 0);
if (showDropoffs) {
// Show users who dropped off at this step (completed this step but not the next)
query.where('level', '=', targetLevel);
} else {
// Show users who completed at least this step
query.where('level', '>=', targetLevel);
}
const profileIdsResult = (await query.execute()) as {
profile_id: string;
}[];
if (profileIdsResult.length === 0) {
return [];
}
// Fetch profile details
const ids = profileIdsResult.map((p) => p.profile_id).filter(Boolean);
const profiles = await getProfilesCached(ids, projectId);
return profiles;
}),
});
function processCohortData(

4
packages/trpc/src/routers/report.ts
View File

@@ -46,7 +46,7 @@ export const reportRouter = createTRPCRouter({
projectId: dashboard.projectId,
dashboardId,
name: report.name,
events: report.events,
events: report.series,
interval: report.interval,
breakdowns: report.breakdowns,
chartType: report.chartType,
@@ -91,7 +91,7 @@ export const reportRouter = createTRPCRouter({
},
data: {
name: report.name,
events: report.events,
events: report.series,
interval: report.interval,
breakdowns: report.breakdowns,
chartType: report.chartType,