zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: report editor

Browse Source 
commit bfcf271a64c33a60f61f511cec2198d9c8a9c51a
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Wed Nov 26 12:32:40 2025 +0100

    wip

commit 8cd3b89fa3
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 22:33:58 2025 +0100

    funnel

commit 95af86dc44
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 22:23:25 2025 +0100

    wip

commit 727a218e6b
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 10:18:26 2025 +0100

    conversion wip

commit 958ba535d6
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 10:18:20 2025 +0100

    wip

commit 3bbeb927cc
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 09:18:48 2025 +0100

    wip

commit d99335e2f4
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 18:08:10 2025 +0100

    wip

commit 1fa61b1ae9
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 15:50:28 2025 +0100

    ts

commit 548747d826
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 13:17:01 2025 +0100

    fix typecheck events -> series

commit 7b18544085
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 13:06:46 2025 +0100

    fix report table

commit 57697a5a39
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Sat Nov 22 00:05:13 2025 +0100

    wip

commit 06fb6c4f3c
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Fri Nov 21 11:21:17 2025 +0100

    wip

commit dd71fd4e11
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Thu Nov 20 13:56:58 2025 +0100

    formulas
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-26 12:33:41 +01:00
parent 828c8c4f91
commit b421474616
70 changed files with 6867 additions and 1918 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
packages/db/code-migrations/7-migrate-events-to-series.ts Normal file
View File

@@ -0,0 +1,104 @@
import { shortId } from '@openpanel/common';
import type {
IChartEvent,
IChartEventItem,
IChartFormula,
} from '@openpanel/validation';
import { db } from '../index';
import { printBoxMessage } from './helpers';
export async function up() {
printBoxMessage('🔄 Migrating Events to Series Format', []);
// Get all reports
const reports = await db.report.findMany({
select: {
id: true,
events: true,
formula: true,
name: true,
},
});
let migratedCount = 0;
let skippedCount = 0;
let formulaAddedCount = 0;
for (const report of reports) {
const events = report.events as unknown as Array<
Partial<IChartEventItem> | Partial<IChartEvent>
>;
const oldFormula = report.formula;
// Check if any event is missing the 'type' field (old format)
const needsEventMigration =
Array.isArray(events) &&
events.length > 0 &&
events.some(
(event) => !event || typeof event !== 'object' || !('type' in event),
);
// Check if formula exists and isn't already in the series
const hasFormulaInSeries =
Array.isArray(events) &&
events.some(
(item) =>
item &&
typeof item === 'object' &&
'type' in item &&
item.type === 'formula',
);
const needsFormulaMigration = !!oldFormula && !hasFormulaInSeries;
// Skip if no migration needed
if (!needsEventMigration && !needsFormulaMigration) {
skippedCount++;
continue;
}
// Transform events to new format: add type: 'event' to each event
const migratedSeries: IChartEventItem[] = Array.isArray(events)
? events.map((event) => {
if (event && typeof event === 'object' && 'type' in event) {
return event as IChartEventItem;
}
return {
...event,
type: 'event',
} as IChartEventItem;
})
: [];
// Add formula to series if it exists and isn't already there
if (needsFormulaMigration && oldFormula) {
const formulaItem: IChartFormula = {
type: 'formula',
formula: oldFormula,
id: shortId(),
};
migratedSeries.push(formulaItem);
formulaAddedCount++;
}
console.log(
`Updating report ${report.name} (${report.id}) with ${migratedSeries.length} series`,
);
// Update the report with migrated series
await db.report.update({
where: { id: report.id },
data: {
events: migratedSeries,
},
});
migratedCount++;
}
printBoxMessage('✅ Migration Complete', [
`Migrated: ${migratedCount} reports`,
`Formulas added: ${formulaAddedCount} reports`,
`Skipped: ${skippedCount} reports (already in new format or empty)`,
]);
}

1
packages/db/index.ts
View File

@@ -3,6 +3,7 @@ export * from './src/clickhouse/client';
export * from './src/clickhouse/csv';
export * from './src/sql-builder';
export * from './src/services/chart.service';
export * from './src/engine';
export * from './src/services/clients.service';
export * from './src/services/dashboard.service';
export * from './src/services/event.service';

1
packages/db/package.json
View File

@@ -25,6 +25,7 @@
"@prisma/extension-read-replicas": "^0.4.1",
"fast-deep-equal": "^3.1.3",
"jiti": "^2.4.1",
"mathjs": "^12.3.2",
"prisma-json-types-generator": "^3.1.1",
"ramda": "^0.29.1",
"sqlstring": "^2.3.3",

112
packages/db/scripts/ch-copy-from-remote.ts Normal file
View File

@@ -0,0 +1,112 @@
import { stdin as input, stdout as output } from 'node:process';
import { createInterface } from 'node:readline/promises';
import { parseArgs } from 'node:util';
import sqlstring from 'sqlstring';
import { ch } from '../src/clickhouse/client';
import { clix } from '../src/clickhouse/query-builder';
async function main() {
const rl = createInterface({ input, output });
try {
const { values } = parseArgs({
args: process.argv.slice(2),
options: {
host: { type: 'string' },
user: { type: 'string' },
password: { type: 'string' },
db: { type: 'string' },
start: { type: 'string' },
end: { type: 'string' },
projects: { type: 'string' },
},
strict: false,
});
const getArg = (val: unknown): string | undefined =>
typeof val === 'string' ? val : undefined;
console.log('Copy data from remote ClickHouse to local');
console.log('---------------------------------------');
const host =
getArg(values.host) || (await rl.question('Remote Host (IP/Domain): '));
if (!host) throw new Error('Host is required');
const user = getArg(values.user) || (await rl.question('Remote User: '));
if (!user) throw new Error('User is required');
const password =
getArg(values.password) || (await rl.question('Remote Password: '));
if (!password) throw new Error('Password is required');
const dbName =
getArg(values.db) ||
(await rl.question('Remote DB Name (default: openpanel): ')) ||
'openpanel';
const startDate =
getArg(values.start) ||
(await rl.question('Start Date (YYYY-MM-DD HH:mm:ss): '));
if (!startDate) throw new Error('Start date is required');
const endDate =
getArg(values.end) ||
(await rl.question('End Date (YYYY-MM-DD HH:mm:ss): '));
if (!endDate) throw new Error('End date is required');
const projectIdsInput =
getArg(values.projects) ||
(await rl.question(
'Project IDs (comma separated, leave empty for all): ',
));
const projectIds = projectIdsInput
? projectIdsInput.split(',').map((s: string) => s.trim())
: [];
console.log('\nStarting copy process...');
const tables = ['sessions', 'events'];
for (const table of tables) {
console.log(`Processing table: ${table}`);
// Build the SELECT part using the query builder
// We use sqlstring to escape the remote function arguments
const remoteTable = `remote(${sqlstring.escape(host)}, ${sqlstring.escape(dbName)}, ${sqlstring.escape(table)}, ${sqlstring.escape(user)}, ${sqlstring.escape(password)})`;
const queryBuilder = clix(ch)
.from(remoteTable)
.select(['*'])
.where('created_at', 'BETWEEN', [startDate, endDate]);
if (projectIds.length > 0) {
queryBuilder.where('project_id', 'IN', projectIds);
}
const selectQuery = queryBuilder.toSQL();
const insertQuery = `INSERT INTO ${dbName}.${table} ${selectQuery}`;
console.log(`Executing: ${insertQuery}`);
// try {
// await ch.command({
// query: insertQuery,
// });
// console.log(`✅ Copied ${table} successfully`);
// } catch (error) {
// console.error(`❌ Failed to copy ${table}:`, error);
// }
}
console.log('\nDone!');
} catch (error) {
console.error('\nError:', error);
} finally {
rl.close();
await ch.close();
process.exit(0);
}
}
main();

96
packages/db/scripts/ch-update-sessions-with-revenue.ts Normal file
View File

@@ -0,0 +1,96 @@
import { TABLE_NAMES, ch } from '../src/clickhouse/client';
import { clix } from '../src/clickhouse/query-builder';
const START_DATE = new Date('2025-11-10T00:00:00Z');
const END_DATE = new Date('2025-11-20T23:00:00Z');
const SESSIONS_PER_HOUR = 2;
// Revenue between $10 (1000 cents) and $200 (20000 cents)
const MIN_REVENUE = 1000;
const MAX_REVENUE = 20000;
function getRandomRevenue() {
return (
Math.floor(Math.random() * (MAX_REVENUE - MIN_REVENUE + 1)) + MIN_REVENUE
);
}
async function main() {
console.log(
`Starting revenue update for sessions between ${START_DATE.toISOString()} and ${END_DATE.toISOString()}`,
);
let currentDate = new Date(START_DATE);
while (currentDate < END_DATE) {
const nextHour = new Date(currentDate.getTime() + 60 * 60 * 1000);
console.log(`Processing hour: ${currentDate.toISOString()}`);
// 1. Pick random sessions for this hour
const sessions = await clix(ch)
.from(TABLE_NAMES.sessions)
.select(['id'])
.where('created_at', '>=', currentDate)
.andWhere('created_at', '<', nextHour)
.where('project_id', '=', 'public-web')
.limit(SESSIONS_PER_HOUR)
.execute();
if (sessions.length === 0) {
console.log(`No sessions found for ${currentDate.toISOString()}`);
currentDate = nextHour;
continue;
}
const sessionIds = sessions.map((s: any) => s.id);
console.log(
`Found ${sessionIds.length} sessions to update: ${sessionIds.join(', ')}`,
);
// 2. Construct update query
// We want to assign a DIFFERENT random revenue to each session
// Query: ALTER TABLE sessions UPDATE revenue = if(id='id1', rev1, if(id='id2', rev2, ...)) WHERE id IN ('id1', 'id2', ...)
const updates: { id: string; revenue: number }[] = [];
for (const id of sessionIds) {
const revenue = getRandomRevenue();
updates.push({ id, revenue });
}
// Build nested if() for the update expression
// ClickHouse doesn't have CASE WHEN in UPDATE expression in the same way, but if() works.
// Actually multiIf is cleaner: multiIf(id='id1', rev1, id='id2', rev2, revenue)
const conditions = updates
.map((u) => `id = '${u.id}', ${u.revenue}`)
.join(', ');
const updateExpr = `multiIf(${conditions}, revenue)`;
const idsStr = sessionIds.map((id: string) => `'${id}'`).join(', ');
const query = `ALTER TABLE ${TABLE_NAMES.sessions} UPDATE revenue = ${updateExpr} WHERE id IN (${idsStr})`;
console.log(`Executing update: ${query}`);
try {
await ch.command({
query,
});
console.log('Update command sent.');
// Wait a bit to not overload mutations if running on a large range
await new Promise((resolve) => setTimeout(resolve, 500));
} catch (error) {
console.error('Failed to update sessions:', error);
}
currentDate = nextHour;
}
console.log('Done!');
}
main().catch((error) => {
console.error('Script failed:', error);
process.exit(1);
});

1
packages/db/src/clickhouse/query-builder.ts
View File

@@ -731,6 +731,7 @@ clix.toInterval = (node: string, interval: IInterval) => {
};
clix.toDate = (node: string, interval: IInterval) => {
switch (interval) {
case 'day':
case 'week':
case 'month': {
return `toDate(${node})`;

216
packages/db/src/engine/compute.ts Normal file
View File

@@ -0,0 +1,216 @@
import { round } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import type { IChartFormula } from '@openpanel/validation';
import * as mathjs from 'mathjs';
import type { ConcreteSeries } from './types';
/**
* Compute formula series from fetched event series
* Formulas reference event series using alphabet IDs (A, B, C, etc.)
*/
export function compute(
fetchedSeries: ConcreteSeries[],
definitions: Array<{
type: 'event' | 'formula';
id?: string;
formula?: string;
}>,
): ConcreteSeries[] {
const results: ConcreteSeries[] = [...fetchedSeries];
// Process formulas in order (they can reference previous formulas)
definitions.forEach((definition, formulaIndex) => {
if (definition.type !== 'formula') {
return;
}
const formula = definition as IChartFormula;
if (!formula.formula) {
return;
}
// Group ALL series (events + previously computed formulas) by breakdown signature
// Series with the same breakdown values should be computed together
const seriesByBreakdown = new Map<string, ConcreteSeries[]>();
// Include both fetched event series AND previously computed formulas
const allSeries = [
...fetchedSeries,
...results.filter((s) => s.definitionIndex < formulaIndex),
];
allSeries.forEach((serie) => {
// Create breakdown signature: skip first name part (event/formula name) and use breakdown values
// If name.length === 1, it means no breakdowns (just event name)
// If name.length > 1, name[0] is event name, name[1+] are breakdown values
const breakdownSignature =
serie.name.length > 1 ? serie.name.slice(1).join(':::') : '';
if (!seriesByBreakdown.has(breakdownSignature)) {
seriesByBreakdown.set(breakdownSignature, []);
}
seriesByBreakdown.get(breakdownSignature)!.push(serie);
});
// Compute formula for each breakdown group
for (const [breakdownSignature, breakdownSeries] of seriesByBreakdown) {
// Map series by their definition index for formula evaluation
const seriesByIndex = new Map<number, ConcreteSeries>();
breakdownSeries.forEach((serie) => {
seriesByIndex.set(serie.definitionIndex, serie);
});
// Get all unique dates across all series in this breakdown group
const allDates = new Set<string>();
breakdownSeries.forEach((serie) => {
serie.data.forEach((item) => {
allDates.add(item.date);
});
});
const sortedDates = Array.from(allDates).sort(
(a, b) => new Date(a).getTime() - new Date(b).getTime(),
);
// Calculate total_count for the formula using the same formula applied to input series' total_count values
// total_count is constant across all dates for a breakdown group, so compute it once
const totalCountScope: Record<string, number> = {};
definitions.slice(0, formulaIndex).forEach((depDef, depIndex) => {
const readableId = alphabetIds[depIndex];
if (!readableId) {
return;
}
// Find the series for this dependency in the current breakdown group
const depSeries = seriesByIndex.get(depIndex);
if (depSeries) {
// Get total_count from any data point (it's the same for all dates)
const totalCount = depSeries.data.find(
(d) => d.total_count != null,
)?.total_count;
totalCountScope[readableId] = totalCount ?? 0;
} else {
// Could be a formula from a previous breakdown group - find it in results
const formulaSerie = results.find(
(s) =>
s.definitionIndex === depIndex &&
'type' in s.definition &&
s.definition.type === 'formula' &&
s.name.slice(1).join(':::') === breakdownSignature,
);
if (formulaSerie) {
const totalCount = formulaSerie.data.find(
(d) => d.total_count != null,
)?.total_count;
totalCountScope[readableId] = totalCount ?? 0;
} else {
totalCountScope[readableId] = 0;
}
}
});
// Evaluate formula for total_count
let formulaTotalCount: number | undefined;
try {
const result = mathjs
.parse(formula.formula)
.compile()
.evaluate(totalCountScope) as number;
formulaTotalCount =
Number.isNaN(result) || !Number.isFinite(result)
? undefined
: round(result, 2);
} catch (error) {
formulaTotalCount = undefined;
}
// Calculate formula for each date
const formulaData = sortedDates.map((date) => {
const scope: Record<string, number> = {};
// Build scope using alphabet IDs (A, B, C, etc.)
definitions.slice(0, formulaIndex).forEach((depDef, depIndex) => {
const readableId = alphabetIds[depIndex];
if (!readableId) {
return;
}
// Find the series for this dependency in the current breakdown group
const depSeries = seriesByIndex.get(depIndex);
if (depSeries) {
const dataPoint = depSeries.data.find((d) => d.date === date);
scope[readableId] = dataPoint?.count ?? 0;
} else {
// Could be a formula from a previous breakdown group - find it in results
// Match by definitionIndex AND breakdown signature
const formulaSerie = results.find(
(s) =>
s.definitionIndex === depIndex &&
'type' in s.definition &&
s.definition.type === 'formula' &&
s.name.slice(1).join(':::') === breakdownSignature,
);
if (formulaSerie) {
const dataPoint = formulaSerie.data.find((d) => d.date === date);
scope[readableId] = dataPoint?.count ?? 0;
} else {
scope[readableId] = 0;
}
}
});
// Evaluate formula
let count: number;
try {
count = mathjs
.parse(formula.formula)
.compile()
.evaluate(scope) as number;
} catch (error) {
count = 0;
}
return {
date,
count:
Number.isNaN(count) || !Number.isFinite(count)
? 0
: round(count, 2),
total_count: formulaTotalCount,
};
});
// Create concrete series for this formula
const templateSerie = breakdownSeries[0]!;
// Extract breakdown values from template series name
// name[0] is event/formula name, name[1+] are breakdown values
const breakdownValues =
templateSerie.name.length > 1 ? templateSerie.name.slice(1) : [];
const formulaName =
breakdownValues.length > 0
? [formula.displayName || formula.formula, ...breakdownValues]
: [formula.displayName || formula.formula];
const formulaSeries: ConcreteSeries = {
id: `formula-${formula.id ?? formulaIndex}-${breakdownSignature || 'default'}`,
definitionId:
formula.id ?? alphabetIds[formulaIndex] ?? `formula-${formulaIndex}`,
definitionIndex: formulaIndex,
name: formulaName,
context: {
filters: templateSerie.context.filters,
breakdownValue: templateSerie.context.breakdownValue,
breakdowns: templateSerie.context.breakdowns,
},
data: formulaData,
definition: formula,
};
results.push(formulaSeries);
}
});
return results;
}

151
packages/db/src/engine/fetch.ts Normal file
View File

@@ -0,0 +1,151 @@
import type { ISerieDataItem } from '@openpanel/common';
import { groupByLabels } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import type { IGetChartDataInput } from '@openpanel/validation';
import { chQuery } from '../clickhouse/client';
import { getChartSql } from '../services/chart.service';
import type { ConcreteSeries, Plan } from './types';
/**
* Fetch data for all event series in the plan
* This handles breakdown expansion automatically via groupByLabels
*/
export async function fetch(plan: Plan): Promise<ConcreteSeries[]> {
const results: ConcreteSeries[] = [];
// Process each event definition
for (let i = 0; i < plan.definitions.length; i++) {
const definition = plan.definitions[i]!;
if (definition.type !== 'event') {
// Skip formulas - they'll be handled in compute stage
continue;
}
const event = definition as typeof definition & { type: 'event' };
// Find the corresponding concrete series placeholder
const placeholder = plan.concreteSeries.find(
(cs) => cs.definitionId === definition.id,
);
if (!placeholder) {
continue;
}
// Build query input
const queryInput: IGetChartDataInput = {
event: {
id: event.id,
name: event.name,
segment: event.segment,
filters: event.filters,
displayName: event.displayName,
property: event.property,
},
projectId: plan.input.projectId,
startDate: plan.input.startDate,
endDate: plan.input.endDate,
breakdowns: plan.input.breakdowns,
interval: plan.input.interval,
chartType: plan.input.chartType,
metric: plan.input.metric,
previous: plan.input.previous ?? false,
limit: plan.input.limit,
offset: plan.input.offset,
criteria: plan.input.criteria,
funnelGroup: plan.input.funnelGroup,
funnelWindow: plan.input.funnelWindow,
};
// Execute query
let queryResult = await chQuery<ISerieDataItem>(
getChartSql({ ...queryInput, timezone: plan.timezone }),
{
session_timezone: plan.timezone,
},
);
// Fallback: if no results with breakdowns, try without breakdowns
if (queryResult.length === 0 && plan.input.breakdowns.length > 0) {
queryResult = await chQuery<ISerieDataItem>(
getChartSql({
...queryInput,
breakdowns: [],
timezone: plan.timezone,
}),
{
session_timezone: plan.timezone,
},
);
}
// Group by labels (handles breakdown expansion)
const groupedSeries = groupByLabels(queryResult);
// Create concrete series for each grouped result
groupedSeries.forEach((grouped) => {
// Extract breakdown value from name array
// If breakdowns exist, name[0] is event name, name[1+] are breakdown values
const breakdownValue =
plan.input.breakdowns.length > 0 && grouped.name.length > 1
? grouped.name.slice(1).join(' - ')
: undefined;
// Build breakdowns object: { country: 'SE', path: '/ewoqmepwq' }
const breakdowns: Record<string, string> | undefined =
plan.input.breakdowns.length > 0 && grouped.name.length > 1
? {}
: undefined;
if (breakdowns) {
plan.input.breakdowns.forEach((breakdown, idx) => {
const breakdownNamePart = grouped.name[idx + 1];
if (breakdownNamePart) {
breakdowns[breakdown.name] = breakdownNamePart;
}
});
}
// Build filters including breakdown value
const filters = [...event.filters];
if (breakdownValue && plan.input.breakdowns.length > 0) {
// Add breakdown filter
plan.input.breakdowns.forEach((breakdown, idx) => {
const breakdownNamePart = grouped.name[idx + 1];
if (breakdownNamePart) {
filters.push({
id: `breakdown-${idx}`,
name: breakdown.name,
operator: 'is',
value: [breakdownNamePart],
});
}
});
}
const concrete: ConcreteSeries = {
id: `${placeholder.id}-${grouped.name.join('-')}`,
definitionId: definition.id ?? alphabetIds[i] ?? `series-${i}`,
definitionIndex: i,
name: grouped.name,
context: {
event: event.name,
filters,
breakdownValue,
breakdowns,
},
data: grouped.data.map((item) => ({
date: item.date,
count: item.count,
total_count: item.total_count,
})),
definition,
};
results.push(concrete);
});
}
return results;
}

145
packages/db/src/engine/format.ts Normal file
View File

@@ -0,0 +1,145 @@
import {
average,
getPreviousMetric,
max,
min,
round,
slug,
sum,
} from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import type { FinalChart } from '@openpanel/validation';
import type { ConcreteSeries } from './types';
/**
* Format concrete series into FinalChart format (backward compatible)
* TODO: Migrate frontend to use cleaner ChartResponse format
*/
export function format(
concreteSeries: ConcreteSeries[],
definitions: Array<{
id?: string;
type: 'event' | 'formula';
displayName?: string;
formula?: string;
name?: string;
}>,
includeAlphaIds: boolean,
previousSeries: ConcreteSeries[] | null = null,
limit: number | undefined = undefined,
): FinalChart {
const series = concreteSeries.map((cs) => {
// Find definition for this series
const definition = definitions[cs.definitionIndex];
const alphaId = includeAlphaIds
? alphabetIds[cs.definitionIndex]
: undefined;
// Build display name with optional alpha ID
let displayName: string[];
// Replace the first name (which is the event name) with the display name if it exists
const names = cs.name.slice(0);
if (cs.definition.displayName) {
names.splice(0, 1, cs.definition.displayName);
}
// Add the alpha ID to the first name if it exists
if (alphaId) {
displayName = [`(${alphaId}) ${names[0]}`, ...names.slice(1)];
} else {
displayName = names;
}
// Calculate metrics for this series
const counts = cs.data.map((d) => d.count);
const metrics = {
sum: sum(counts),
average: round(average(counts), 2),
min: min(counts),
max: max(counts),
count: cs.data.find((item) => !!item.total_count)?.total_count,
};
// Build event object for compatibility
const eventName =
definition?.type === 'formula'
? definition.displayName || definition.formula || 'Formula'
: definition?.name || cs.context.event || 'unknown';
// Find matching previous series
const previousSerie = previousSeries?.find(
(ps) =>
ps.definitionIndex === cs.definitionIndex &&
ps.name.slice(1).join(':::') === cs.name.slice(1).join(':::'),
);
return {
id: slug(cs.id),
names: displayName,
// TODO: Do we need this now?
event: {
id: definition?.id,
name: eventName,
breakdowns: cs.context.breakdowns,
},
metrics: {
...metrics,
...(previousSerie
? {
previous: {
sum: getPreviousMetric(
metrics.sum,
sum(previousSerie.data.map((d) => d.count)),
),
average: getPreviousMetric(
metrics.average,
round(average(previousSerie.data.map((d) => d.count)), 2),
),
min: getPreviousMetric(
metrics.min,
min(previousSerie.data.map((d) => d.count)),
),
max: getPreviousMetric(
metrics.max,
max(previousSerie.data.map((d) => d.count)),
),
count: getPreviousMetric(
metrics.count ?? 0,
previousSerie.data.find((item) => !!item.total_count)
?.total_count ?? null,
),
},
}
: {}),
},
data: cs.data.map((item, index) => ({
date: item.date,
count: item.count,
previous: previousSerie?.data[index]
? getPreviousMetric(
item.count,
previousSerie.data[index]?.count ?? null,
)
: undefined,
})),
};
});
// Sort series by sum (biggest first)
series.sort((a, b) => b.metrics.sum - a.metrics.sum);
// Calculate global metrics
const allValues = concreteSeries.flatMap((cs) => cs.data.map((d) => d.count));
const globalMetrics = {
sum: sum(allValues),
average: round(average(allValues), 2),
min: min(allValues),
max: max(allValues),
count: undefined as number | undefined,
};
return {
series: limit ? series.slice(0, limit) : series,
metrics: globalMetrics,
};
}

75
packages/db/src/engine/index.ts Normal file
View File

@@ -0,0 +1,75 @@
import { getPreviousMetric } from '@openpanel/common';
import type { FinalChart, IChartInput } from '@openpanel/validation';
import { getChartPrevStartEndDate } from '../services/chart.service';
import {
getOrganizationSubscriptionChartEndDate,
getSettingsForProject,
} from '../services/organization.service';
import { compute } from './compute';
import { fetch } from './fetch';
import { format } from './format';
import { normalize } from './normalize';
import { plan } from './plan';
import type { ConcreteSeries } from './types';
/**
* Chart Engine - Main entry point
* Executes the pipeline: normalize -> plan -> fetch -> compute -> format
*/
export async function executeChart(input: IChartInput): Promise<FinalChart> {
// Stage 1: Normalize input
const normalized = await normalize(input);
// Handle subscription end date limit
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
normalized.endDate,
);
if (endDate) {
normalized.endDate = endDate;
}
// Stage 2: Create execution plan
const executionPlan = await plan(normalized);
// Stage 3: Fetch data for event series (current period)
const fetchedSeries = await fetch(executionPlan);
// Stage 4: Compute formula series
const computedSeries = compute(fetchedSeries, executionPlan.definitions);
// Stage 5: Fetch previous period if requested
let previousSeries: ConcreteSeries[] | null = null;
if (input.previous) {
const currentPeriod = {
startDate: normalized.startDate,
endDate: normalized.endDate,
};
const previousPeriod = getChartPrevStartEndDate(currentPeriod);
const previousPlan = await plan({
...normalized,
...previousPeriod,
});
const previousFetched = await fetch(previousPlan);
previousSeries = compute(previousFetched, previousPlan.definitions);
}
// Stage 6: Format final output with previous period data
const includeAlphaIds = executionPlan.definitions.length > 1;
const response = format(
computedSeries,
executionPlan.definitions,
includeAlphaIds,
previousSeries,
);
return response;
}
// Export as ChartEngine for backward compatibility
export const ChartEngine = {
execute: executeChart,
};

66
packages/db/src/engine/normalize.ts Normal file
View File

@@ -0,0 +1,66 @@
import { alphabetIds } from '@openpanel/constants';
import type {
IChartEvent,
IChartEventItem,
IChartInput,
IChartInputWithDates,
} from '@openpanel/validation';
import { getChartStartEndDate } from '../services/chart.service';
import { getSettingsForProject } from '../services/organization.service';
import type { SeriesDefinition } from './types';
export type NormalizedInput = Awaited<ReturnType<typeof normalize>>;
/**
* Normalize a chart input into a clean structure with dates and normalized series
*/
export async function normalize(
input: IChartInput,
): Promise<IChartInputWithDates & { series: SeriesDefinition[] }> {
const { timezone } = await getSettingsForProject(input.projectId);
const { startDate, endDate } = getChartStartEndDate(
{
range: input.range,
startDate: input.startDate ?? undefined,
endDate: input.endDate ?? undefined,
},
timezone,
);
// Get series from input (handles both 'series' and 'events' fields)
// The schema preprocessing should have already converted 'events' to 'series', but handle both for safety
const rawSeries = (input as any).series ?? (input as any).events ?? [];
// Normalize each series item
const normalizedSeries: SeriesDefinition[] = rawSeries.map(
(item: any, index: number) => {
// If item already has type field, it's the new format
if (item && typeof item === 'object' && 'type' in item) {
return {
...item,
id: item.id ?? alphabetIds[index] ?? `series-${index}`,
} as SeriesDefinition;
}
// Old format without type field - assume it's an event
const event = item as Partial<IChartEvent>;
return {
type: 'event',
id: event.id ?? alphabetIds[index] ?? `series-${index}`,
name: event.name || 'unknown_event',
segment: event.segment ?? 'event',
filters: event.filters ?? [],
displayName: event.displayName,
property: event.property,
} as SeriesDefinition;
},
);
return {
...input,
series: normalizedSeries,
startDate,
endDate,
};
}

50
packages/db/src/engine/plan.ts Normal file
View File

@@ -0,0 +1,50 @@
import { slug } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import type { IChartEventItem } from '@openpanel/validation';
import { getSettingsForProject } from '../services/organization.service';
import type { NormalizedInput } from './normalize';
import type { ConcreteSeries, Plan } from './types';
/**
* Create an execution plan from normalized input
* This sets up ConcreteSeries placeholders - actual breakdown expansion happens during fetch
*/
export async function plan(normalized: NormalizedInput): Promise<Plan> {
const { timezone } = await getSettingsForProject(normalized.projectId);
const concreteSeries: ConcreteSeries[] = [];
// Create concrete series placeholders for each definition
normalized.series.forEach((definition, index) => {
if (definition.type === 'event') {
const event = definition as IChartEventItem & { type: 'event' };
// For events, create a placeholder
// If breakdowns exist, fetch will return multiple series (one per breakdown value)
// If no breakdowns, fetch will return one series
const concrete: ConcreteSeries = {
id: `${slug(event.name)}-${event.id ?? index}`,
definitionId: event.id ?? alphabetIds[index] ?? `series-${index}`,
definitionIndex: index,
name: [event.displayName || event.name],
context: {
event: event.name,
filters: [...event.filters],
},
data: [], // Will be populated by fetch stage
definition,
};
concreteSeries.push(concrete);
} else {
// For formulas, we'll create placeholders during compute stage
// Formulas depend on event series, so we skip them here
}
});
return {
concreteSeries,
definitions: normalized.series,
input: normalized,
timezone,
};
}

85
packages/db/src/engine/types.ts Normal file
View File

@@ -0,0 +1,85 @@
import type {
IChartBreakdown,
IChartEvent,
IChartEventFilter,
IChartEventItem,
IChartFormula,
IChartInput,
IChartInputWithDates,
} from '@openpanel/validation';
/**
* Series Definition - The input representation of what the user wants
* This is what comes from the frontend (events or formulas)
*/
export type SeriesDefinition = IChartEventItem;
/**
* Concrete Series - A resolved series that will be displayed as a line/bar on the chart
* When breakdowns exist, one SeriesDefinition can expand into multiple ConcreteSeries
*/
export type ConcreteSeries = {
id: string;
definitionId: string; // ID of the SeriesDefinition this came from
definitionIndex: number; // Index in the original series array (for A, B, C references)
name: string[]; // Display name parts: ["Session Start", "Chrome"] or ["Formula 1"]
// Context for Drill-down / Profiles
// This contains everything needed to query 'who are these users?'
context: {
event?: string; // Event name (if this is an event series)
filters: IChartEventFilter[]; // All filters including breakdown value
breakdownValue?: string; // The breakdown value for this concrete series (deprecated, use breakdowns instead)
breakdowns?: Record<string, string>; // Breakdown keys and values: { country: 'SE', path: '/ewoqmepwq' }
};
// Data points for this series
data: Array<{
date: string;
count: number;
total_count?: number;
}>;
// The original definition (event or formula)
definition: SeriesDefinition;
};
/**
* Plan - The execution plan after normalization and expansion
*/
export type Plan = {
concreteSeries: ConcreteSeries[];
definitions: SeriesDefinition[];
input: IChartInputWithDates;
timezone: string;
};
/**
* Chart Response - The final output format
*/
export type ChartResponse = {
series: Array<{
id: string;
name: string[];
data: Array<{
date: string;
value: number;
previous?: number;
}>;
summary: {
total: number;
average: number;
min: number;
max: number;
count?: number;
};
context?: ConcreteSeries['context']; // Include context for drill-down
}>;
summary: {
total: number;
average: number;
min: number;
max: number;
};
};

76
packages/db/src/services/chart.service.ts
View File

@@ -155,7 +155,8 @@ export function getChartSql({
}
breakdowns.forEach((breakdown, index) => {
const key = `label_${index}`;
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -175,8 +176,8 @@ export function getChartSql({
if (event.segment === 'property_sum' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `sum(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'sum(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `sum(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -185,8 +186,8 @@ export function getChartSql({
if (event.segment === 'property_average' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `avg(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'avg(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `avg(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -195,8 +196,8 @@ export function getChartSql({
if (event.segment === 'property_max' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `max(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'max(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `max(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -205,8 +206,8 @@ export function getChartSql({
if (event.segment === 'property_min' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `min(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'min(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `min(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -230,14 +231,58 @@ export function getChartSql({
return sql;
}
// Add total unique count for user segment using a scalar subquery
if (event.segment === 'user') {
const totalUniqueSubquery = `(
SELECT ${sb.select.count}
// Build total_count calculation that accounts for breakdowns
// When breakdowns exist, we need to calculate total_count per breakdown group
if (breakdowns.length > 0) {
// Create a subquery that calculates total_count per breakdown group (without date grouping)
// Then reference it in the main query via JOIN
const breakdownSelects = breakdowns
.map((breakdown, index) => {
const key = `label_${index + 1}`;
const breakdownExpr = getSelectPropertyKey(breakdown.name);
return `${breakdownExpr} as ${key}`;
})
.join(', ');
// GROUP BY needs to use the actual expressions, not aliases
const breakdownGroupByExprs = breakdowns
.map((breakdown) => getSelectPropertyKey(breakdown.name))
.join(', ');
// Build the total_count subquery grouped only by breakdowns (no date)
// Extract the count expression without the alias (remove "as count")
const countExpression = sb.select.count.replace(/\s+as\s+count$/i, '');
const totalCountSubquery = `(
SELECT
${breakdownSelects},
${countExpression} as total_count
FROM ${sb.from}
${getJoins()}
${getWhere()}
)`;
GROUP BY ${breakdownGroupByExprs}
) as total_counts`;
// Join the total_counts subquery to get total_count per breakdown
// Match on the breakdown column values
const joinConditions = breakdowns
.map((_, index) => {
const outerKey = `label_${index + 1}`;
return `${outerKey} = total_counts.label_${index + 1}`;
})
.join(' AND ');
sb.joins.total_counts = `LEFT JOIN ${totalCountSubquery} ON ${joinConditions}`;
// Use any() aggregate since total_count is the same for all rows in a breakdown group
sb.select.total_unique_count =
'any(total_counts.total_count) as total_count';
} else {
// No breakdowns - use a simple subquery for total count
const totalUniqueSubquery = `(
SELECT ${sb.select.count}
FROM ${sb.from}
${getJoins()}
${getWhere()}
)`;
sb.select.total_unique_count = `${totalUniqueSubquery} as total_count`;
}
@@ -509,12 +554,11 @@ export function getChartStartEndDate(
}: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>,
timezone: string,
) {
const ranges = getDatesFromRange(range, timezone);
if (startDate && endDate) {
return { startDate: startDate, endDate: endDate };
}
const ranges = getDatesFromRange(range, timezone);
if (!startDate && endDate) {
return { startDate: ranges.startDate, endDate: endDate };
}

39
packages/db/src/services/conversion.service.ts
View File

@@ -1,5 +1,5 @@
import { NOT_SET_VALUE } from '@openpanel/constants';
import type { IChartInput } from '@openpanel/validation';
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { omit } from 'ramda';
import { TABLE_NAMES, ch } from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
@@ -7,6 +7,7 @@ import {
getEventFiltersWhereClause,
getSelectPropertyKey,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
export class ConversionService {
constructor(private client: typeof ch) {}
@@ -17,8 +18,9 @@ export class ConversionService {
endDate,
funnelGroup,
funnelWindow = 24,
events,
series,
breakdowns = [],
limit,
interval,
timezone,
}: Omit<IChartInput, 'range' | 'previous' | 'metric' | 'chartType'> & {
@@ -30,6 +32,8 @@ export class ConversionService {
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
const events = onlyReportEvents(series);
if (events.length !== 2) {
throw new Error('events must be an array of two events');
}
@@ -111,18 +115,20 @@ export class ConversionService {
}
const results = await query.execute();
return this.toSeries(results, breakdowns).map((serie, serieIndex) => {
return {
...serie,
data: serie.data.map((d, index) => ({
...d,
timestamp: new Date(d.date).getTime(),
serieIndex,
index,
serie: omit(['data'], serie),
})),
};
});
return this.toSeries(results, breakdowns, limit).map(
(serie, serieIndex) => {
return {
...serie,
data: serie.data.map((d, index) => ({
...d,
timestamp: new Date(d.date).getTime(),
serieIndex,
index,
serie: omit(['data'], serie),
})),
};
},
);
}
private toSeries(
@@ -134,6 +140,7 @@ export class ConversionService {
[key: string]: string | number;
}[],
breakdowns: { name: string }[] = [],
limit: number | undefined = undefined,
) {
if (!breakdowns.length) {
return [
@@ -153,6 +160,10 @@ export class ConversionService {
// Group by breakdown values
const series = data.reduce(
(acc, d) => {
if (limit && Object.keys(acc).length >= limit) {
return acc;
}
const key =
breakdowns.map((b, index) => d[`b_${index}`]).join('|') ||
NOT_SET_VALUE;

149
packages/db/src/services/funnel.service.ts
View File

@@ -1,5 +1,9 @@
import { ifNaN } from '@openpanel/common';
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import type {
IChartEvent,
IChartEventItem,
IChartInput,
} from '@openpanel/validation';
import { last, reverse, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { ch } from '../clickhouse/client';
@@ -10,17 +14,18 @@ import {
getEventFiltersWhereClause,
getSelectPropertyKey,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
export class FunnelService {
constructor(private client: typeof ch) {}
private getFunnelGroup(group?: string) {
getFunnelGroup(group?: string): [string, string] {
return group === 'profile_id'
? [`COALESCE(nullIf(s.pid, ''), profile_id)`, 'profile_id']
: ['session_id', 'session_id'];
}
private getFunnelConditions(events: IChartEvent[]) {
getFunnelConditions(events: IChartEvent[] = []): string[] {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
@@ -29,6 +34,70 @@ export class FunnelService {
});
}
buildFunnelCte({
projectId,
startDate,
endDate,
eventSeries,
funnelWindowMilliseconds,
group,
timezone,
additionalSelects = [],
additionalGroupBy = [],
}: {
projectId: string;
startDate: string;
endDate: string;
eventSeries: IChartEvent[];
funnelWindowMilliseconds: number;
group: [string, string];
timezone: string;
additionalSelects?: string[];
additionalGroupBy?: string[];
}) {
const funnels = this.getFunnelConditions(eventSeries);
return clix(this.client, timezone)
.select([
`${group[0]} AS ${group[1]}`,
...additionalSelects,
`windowFunnel(${funnelWindowMilliseconds}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), ${funnels.join(', ')}) AS level`,
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.where(
'name',
'IN',
eventSeries.map((e) => e.name),
)
.groupBy([group[1], ...additionalGroupBy]);
}
buildSessionsCte({
projectId,
startDate,
endDate,
timezone,
}: {
projectId: string;
startDate: string;
endDate: string;
timezone: string;
}) {
return clix(this.client, timezone)
.select(['profile_id as pid', 'id as sid'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
]);
}
private fillFunnel(
funnel: { level: number; count: number }[],
steps: number,
@@ -57,6 +126,7 @@ export class FunnelService {
toSeries(
funnel: { level: number; count: number; [key: string]: any }[],
breakdowns: { name: string }[] = [],
limit: number | undefined = undefined,
) {
if (!breakdowns.length) {
return [
@@ -72,6 +142,10 @@ export class FunnelService {
// Group by breakdown values
const series = funnel.reduce(
(acc, f) => {
if (limit && Object.keys(acc).length >= limit) {
return acc;
}
const key = breakdowns.map((b, index) => f[`b_${index}`]).join('|');
if (!acc[key]) {
acc[key] = [];
@@ -110,51 +184,49 @@ export class FunnelService {
projectId,
startDate,
endDate,
events,
series,
funnelWindow = 24,
funnelGroup,
breakdowns = [],
limit,
timezone = 'UTC',
}: IChartInput & { timezone: string }) {
}: IChartInput & { timezone: string; events?: IChartEvent[] }) {
if (!startDate || !endDate) {
throw new Error('startDate and endDate are required');
}
if (events.length === 0) {
const eventSeries = onlyReportEvents(series);
if (eventSeries.length === 0) {
throw new Error('events are required');
}
const funnelWindowSeconds = funnelWindow * 3600;
const funnelWindowMilliseconds = funnelWindowSeconds * 1000;
const group = this.getFunnelGroup(funnelGroup);
const funnels = this.getFunnelConditions(events);
const profileFilters = this.getProfileFilters(events);
const profileFilters = this.getProfileFilters(eventSeries);
const anyFilterOnProfile = profileFilters.length > 0;
const anyBreakdownOnProfile = breakdowns.some((b) =>
b.name.startsWith('profile.'),
);
// Create the funnel CTE
const funnelCte = clix(this.client, timezone)
.select([
`${group[0]} AS ${group[1]}`,
...breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
),
`windowFunnel(${funnelWindowMilliseconds}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), ${funnels.join(', ')}) AS level`,
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.where(
'name',
'IN',
events.map((e) => e.name),
)
.groupBy([group[1], ...breakdowns.map((b, index) => `b_${index}`)]);
const breakdownSelects = breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
const funnelCte = this.buildFunnelCte({
projectId,
startDate,
endDate,
eventSeries,
funnelWindowMilliseconds,
group,
timezone,
additionalSelects: breakdownSelects,
additionalGroupBy: breakdownGroupBy,
});
if (anyFilterOnProfile || anyBreakdownOnProfile) {
funnelCte.leftJoin(
@@ -167,15 +239,12 @@ export class FunnelService {
// Create the sessions CTE if needed
const sessionsCte =
group[0] !== 'session_id'
? clix(this.client, timezone)
// Important to have unique field names to avoid ambiguity in the main query
.select(['profile_id as pid', 'id as sid'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
? this.buildSessionsCte({
projectId,
startDate,
endDate,
timezone,
})
: null;
// Base funnel query with CTEs
@@ -204,11 +273,11 @@ export class FunnelService {
.orderBy('level', 'DESC');
const funnelData = await funnelQuery.execute();
const funnelSeries = this.toSeries(funnelData, breakdowns);
const funnelSeries = this.toSeries(funnelData, breakdowns, limit);
return funnelSeries
.map((data) => {
const maxLevel = events.length;
const maxLevel = eventSeries.length;
const filledFunnelRes = this.fillFunnel(
data.map((d) => ({ level: d.level, count: d.count })),
maxLevel,
@@ -220,7 +289,7 @@ export class FunnelService {
(acc, item, index, list) => {
const prev = list[index - 1] ?? { count: totalSessions };
const next = list[index + 1];
const event = events[item.level - 1]!;
const event = eventSeries[item.level - 1]!;
return [
...acc,
{

43
packages/db/src/services/reports.service.ts
View File

@@ -5,19 +5,25 @@ import {
} from '@openpanel/constants';
import type {
IChartBreakdown,
IChartEvent,
IChartEventFilter,
IChartEventItem,
IChartLineType,
IChartProps,
IChartRange,
ICriteria,
} from '@openpanel/validation';
import { db } from '../prisma-client';
import type { Report as DbReport, ReportLayout } from '../prisma-client';
import { db } from '../prisma-client';
export type IServiceReport = Awaited<ReturnType<typeof getReportById>>;
export const onlyReportEvents = (
series: NonNullable<IServiceReport>['series'],
) => {
return series.filter((item) => item.type === 'event');
};
export function transformFilter(
filter: Partial<IChartEventFilter>,
index: number,
@@ -31,17 +37,29 @@ export function transformFilter(
};
}
export function transformReportEvent(
event: Partial<IChartEvent>,
export function transformReportEventItem(
item: IChartEventItem,
index: number,
): IChartEvent {
): IChartEventItem {
if (item.type === 'formula') {
// Transform formula
return {
type: 'formula',
id: item.id ?? alphabetIds[index]!,
formula: item.formula || '',
displayName: item.displayName,
};
}
// Transform event with type field
return {
segment: event.segment ?? 'event',
filters: (event.filters ?? []).map(transformFilter),
id: event.id ?? alphabetIds[index]!,
name: event.name || 'unknown_event',
displayName: event.displayName,
property: event.property,
type: 'event',
segment: item.segment ?? 'event',
filters: (item.filters ?? []).map(transformFilter),
id: item.id ?? alphabetIds[index]!,
name: item.name || 'unknown_event',
displayName: item.displayName,
property: item.property,
};
}
@@ -51,7 +69,8 @@ export function transformReport(
return {
id: report.id,
projectId: report.projectId,
events: (report.events as IChartEvent[]).map(transformReportEvent),
series:
(report.events as IChartEventItem[]).map(transformReportEventItem) ?? [],
breakdowns: report.breakdowns as IChartBreakdown[],
chartType: report.chartType,
lineType: (report.lineType as IChartLineType) ?? lineTypes.monotone,

544
packages/trpc/src/routers/chart.helpers.test.ts
View File

@@ -1,544 +0,0 @@
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { describe, expect, it } from 'vitest';
import { withFormula } from './chart.helpers';
// Helper to create a test event
function createEvent(
id: string,
name: string,
displayName?: string,
): IChartEvent {
return {
id,
name,
displayName: displayName ?? '',
segment: 'event',
filters: [],
};
}
const createChartInput = (
rest: Pick<IChartInput, 'events' | 'formula'>,
): IChartInput => {
return {
metric: 'sum',
chartType: 'linear',
interval: 'day',
breakdowns: [],
projectId: '1',
startDate: '2025-01-01',
endDate: '2025-01-01',
range: '30d',
previous: false,
formula: '',
...rest,
};
};
// Helper to create a test series
function createSeries(
name: string[],
event: IChartEvent,
data: Array<{ date: string; count: number }>,
) {
return {
name,
event,
data: data.map((d) => ({ ...d, total_count: d.count })),
};
}
describe('withFormula', () => {
describe('edge cases', () => {
it('should return series unchanged when formula is empty', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
];
const result = withFormula(
createChartInput({ formula: '', events }),
series,
);
expect(result).toEqual(series);
});
it('should return series unchanged when series is empty', () => {
const events = [createEvent('evt1', 'event1')];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
[],
);
expect(result).toEqual([]);
});
it('should return series unchanged when series has no data', () => {
const events = [createEvent('evt1', 'event1')];
const series = [{ name: ['event1'], event: events[0]!, data: [] }];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toEqual(series);
});
});
describe('single event, no breakdown', () => {
it('should apply simple multiplication formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
{ date: '2025-01-02', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toHaveLength(1);
expect(result[0]?.data).toEqual([
{ date: '2025-01-01', count: 1000, total_count: 10 },
{ date: '2025-01-02', count: 2000, total_count: 20 },
]);
});
it('should apply addition formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 5 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+10', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(15);
});
it('should handle division formula', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/10', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(10);
});
it('should handle NaN and Infinity by returning 0', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 0 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/0', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(0);
});
});
describe('single event, with breakdown', () => {
it('should apply formula to each breakdown group', () => {
const events = [createEvent('evt1', 'screen_view')];
const series = [
createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 10 }]),
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*100', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(1000);
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2000);
});
it('should handle multiple breakdown values', () => {
const events = [createEvent('evt1', 'screen_view')];
const series = [
createSeries(['iOS', 'US'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['Android', 'US'], events[0]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A*2', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS', 'US']);
expect(result[0]?.data[0]?.count).toBe(20);
expect(result[1]?.name).toEqual(['Android', 'US']);
expect(result[1]?.data[0]?.count).toBe(40);
});
});
describe('multiple events, no breakdown', () => {
it('should combine two events with division formula', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['screen_view'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
createSeries(['session_start'], events[1]!, [
{ date: '2025-01-01', count: 50 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(1);
expect(result[0]?.data[0]?.count).toBe(2);
});
it('should combine two events with addition formula', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(30);
});
it('should handle three events', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
createEvent('evt3', 'event3'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
createSeries(['event3'], events[2]!, [
{ date: '2025-01-01', count: 30 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B+C', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(60);
});
it('should handle missing data points with 0', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
{ date: '2025-01-02', count: 20 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 5 },
// Missing 2025-01-02
]),
];
const result = withFormula(
createChartInput({ formula: 'A+B', events }),
series,
);
expect(result[0]?.data[0]?.count).toBe(15); // 10 + 5
expect(result[0]?.data[1]?.count).toBe(20); // 20 + 0 (missing)
});
});
describe('multiple events, with breakdown', () => {
it('should match series by breakdown values and apply formula', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
// iOS breakdown
createSeries(['iOS'], events[0]!, [{ date: '2025-01-01', count: 100 }]),
createSeries(['iOS'], events[1]!, [{ date: '2025-01-01', count: 50 }]),
// Android breakdown
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 200 },
]),
createSeries(['Android'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// iOS: 100/50 = 2
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(2);
// Android: 200/100 = 2
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2);
});
it('should handle multiple breakdown values matching', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['iOS', 'US'], events[0]!, [
{ date: '2025-01-01', count: 100 },
]),
createSeries(['iOS', 'US'], events[1]!, [
{ date: '2025-01-01', count: 50 },
]),
createSeries(['Android', 'US'], events[0]!, [
{ date: '2025-01-01', count: 200 },
]),
createSeries(['Android', 'US'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
expect(result[0]?.name).toEqual(['iOS', 'US']);
expect(result[0]?.data[0]?.count).toBe(2);
expect(result[1]?.name).toEqual(['Android', 'US']);
expect(result[1]?.data[0]?.count).toBe(2);
});
it('should handle different date ranges across breakdown groups', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['iOS'], events[0]!, [
{ date: '2025-01-01', count: 100 },
{ date: '2025-01-02', count: 200 },
]),
createSeries(['iOS'], events[1]!, [
{ date: '2025-01-01', count: 50 },
{ date: '2025-01-02', count: 100 },
]),
createSeries(['Android'], events[0]!, [
{ date: '2025-01-01', count: 300 },
// Missing 2025-01-02
]),
createSeries(['Android'], events[1]!, [
{ date: '2025-01-01', count: 150 },
{ date: '2025-01-02', count: 200 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// iOS group
expect(result[0]?.name).toEqual(['iOS']);
expect(result[0]?.data[0]?.count).toBe(2); // 100/50
expect(result[0]?.data[1]?.count).toBe(2); // 200/100
// Android group
expect(result[1]?.name).toEqual(['Android']);
expect(result[1]?.data[0]?.count).toBe(2); // 300/150
expect(result[1]?.data[1]?.count).toBe(0); // 0/200 = 0 (missing A)
});
});
describe('complex formulas', () => {
it('should handle complex expressions', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
createEvent('evt3', 'event3'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 20 },
]),
createSeries(['event3'], events[2]!, [
{ date: '2025-01-01', count: 30 },
]),
];
const result = withFormula(
createChartInput({ formula: '(A+B)*C', events }),
series,
);
// (10+20)*30 = 900
expect(result[0]?.data[0]?.count).toBe(900);
});
it('should handle percentage calculations', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'session_start'),
];
const series = [
createSeries(['screen_view'], events[0]!, [
{ date: '2025-01-01', count: 75 },
]),
createSeries(['session_start'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
];
const result = withFormula(
createChartInput({ formula: '(A/B)*100', events }),
series,
);
// (75/100)*100 = 75
expect(result[0]?.data[0]?.count).toBe(75);
});
});
describe('error handling', () => {
it('should handle invalid formulas gracefully', () => {
const events = [createEvent('evt1', 'event1')];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
];
const result = withFormula(
createChartInput({ formula: 'invalid formula', events }),
series,
);
// Should return 0 for invalid formulas
expect(result[0]?.data[0]?.count).toBe(0);
});
it('should handle division by zero', () => {
const events = [
createEvent('evt1', 'event1'),
createEvent('evt2', 'event2'),
];
const series = [
createSeries(['event1'], events[0]!, [
{ date: '2025-01-01', count: 10 },
]),
createSeries(['event2'], events[1]!, [
{ date: '2025-01-01', count: 0 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
// Division by zero should result in 0 (Infinity -> 0)
expect(result[0]?.data[0]?.count).toBe(0);
});
});
describe('real-world scenario: article hit ratio', () => {
it('should calculate hit ratio per article path', () => {
const events = [
createEvent('evt1', 'screen_view'),
createEvent('evt2', 'article_card_seen'),
];
const series = [
// Article 1
createSeries(['/articles/1'], events[0]!, [
{ date: '2025-01-01', count: 1000 },
]),
createSeries(['/articles/1'], events[1]!, [
{ date: '2025-01-01', count: 100 },
]),
// Article 2
createSeries(['/articles/2'], events[0]!, [
{ date: '2025-01-01', count: 500 },
]),
createSeries(['/articles/2'], events[1]!, [
{ date: '2025-01-01', count: 200 },
]),
];
const result = withFormula(
createChartInput({ formula: 'A/B', events }),
series,
);
expect(result).toHaveLength(2);
// Article 1: 1000/100 = 10
expect(result[0]?.name).toEqual(['/articles/1']);
expect(result[0]?.data[0]?.count).toBe(10);
// Article 2: 500/200 = 2.5
expect(result[1]?.name).toEqual(['/articles/2']);
expect(result[1]?.data[0]?.count).toBe(2.5);
});
});
});

514
packages/trpc/src/routers/chart.helpers.ts
View File

@@ -1,514 +0,0 @@
import * as mathjs from 'mathjs';
import { last, reverse } from 'ramda';
import sqlstring from 'sqlstring';
import type { ISerieDataItem } from '@openpanel/common';
import {
average,
getPreviousMetric,
groupByLabels,
max,
min,
round,
slug,
sum,
} from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import {
TABLE_NAMES,
chQuery,
createSqlBuilder,
formatClickhouseDate,
getChartPrevStartEndDate,
getChartSql,
getChartStartEndDate,
getEventFiltersWhereClause,
getOrganizationSubscriptionChartEndDate,
getSettingsForProject,
} from '@openpanel/db';
import type {
FinalChart,
IChartEvent,
IChartInput,
IChartInputWithDates,
IGetChartDataInput,
} from '@openpanel/validation';
export function withFormula(
{ formula, events }: IChartInput,
series: Awaited<ReturnType<typeof getChartSerie>>,
) {
if (!formula) {
return series;
}
if (!series || series.length === 0) {
return series;
}
if (!series[0]?.data) {
return series;
}
// Formulas always use alphabet IDs (A, B, C, etc.), not event IDs
// Group series by breakdown values (the name array)
// This allows us to match series from different events that have the same breakdown values
// Detect if we have breakdowns: when there are no breakdowns, name arrays contain event names
// When there are breakdowns, name arrays contain breakdown values (not event names)
const hasBreakdowns = series.some(
(serie) =>
serie.name.length > 0 &&
!events.some(
(event) =>
serie.name[0] === event.name || serie.name[0] === event.displayName,
),
);
const seriesByBreakdown = new Map<string, typeof series>();
series.forEach((serie) => {
let breakdownKey: string;
if (hasBreakdowns) {
// With breakdowns: use the entire name array as the breakdown key
// The name array contains breakdown values (e.g., ["iOS"], ["Android"])
breakdownKey = serie.name.join(':::');
} else {
// Without breakdowns: group all series together regardless of event name
// This allows formulas to combine multiple events
breakdownKey = '';
}
if (!seriesByBreakdown.has(breakdownKey)) {
seriesByBreakdown.set(breakdownKey, []);
}
seriesByBreakdown.get(breakdownKey)!.push(serie);
});
// For each breakdown group, apply the formula
const result: typeof series = [];
for (const [breakdownKey, breakdownSeries] of seriesByBreakdown) {
// Group series by event to ensure we have one series per event
const seriesByEvent = new Map<string, (typeof series)[number]>();
breakdownSeries.forEach((serie) => {
const eventId = serie.event.id ?? serie.event.name;
// If we already have a series for this event in this breakdown group, skip it
// (shouldn't happen, but just in case)
if (!seriesByEvent.has(eventId)) {
seriesByEvent.set(eventId, serie);
}
});
// Get all unique dates across all series in this breakdown group
const allDates = new Set<string>();
breakdownSeries.forEach((serie) => {
serie.data.forEach((item) => {
allDates.add(item.date);
});
});
// Sort dates chronologically
const sortedDates = Array.from(allDates).sort(
(a, b) => new Date(a).getTime() - new Date(b).getTime(),
);
// Apply formula for each date, matching series by event index
const formulaData = sortedDates.map((date) => {
const scope: Record<string, number> = {};
// Build scope using alphabet IDs (A, B, C, etc.) for each event
// This matches how formulas are written (e.g., "A*100", "A/B", "A+B-C")
events.forEach((event, eventIndex) => {
const readableId = alphabetIds[eventIndex];
if (!readableId) {
throw new Error('no alphabet id for serie in withFormula');
}
// Find the series for this event in this breakdown group
const eventId = event.id ?? event.name;
const matchingSerie = seriesByEvent.get(eventId);
// Find the data point for this date
// If the series doesn't exist or the date is missing, use 0
const dataPoint = matchingSerie?.data.find((d) => d.date === date);
scope[readableId] = dataPoint?.count ?? 0;
});
// Evaluate the formula with the scope
let count: number;
try {
count = mathjs.parse(formula).compile().evaluate(scope) as number;
} catch (error) {
// If formula evaluation fails, return 0
count = 0;
}
return {
date,
count:
Number.isNaN(count) || !Number.isFinite(count) ? 0 : round(count, 2),
total_count: breakdownSeries[0]?.data.find((d) => d.date === date)
?.total_count,
};
});
// Use the first series as a template, but replace its data with formula results
// Preserve the breakdown labels (name array) from the original series
const templateSerie = breakdownSeries[0]!;
result.push({
...templateSerie,
data: formulaData,
});
}
return result;
}
function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
const filled = Array.from({ length: steps }, (_, index) => {
const level = index + 1;
const matchingResult = funnel.find((res) => res.level === level);
return {
level,
count: matchingResult ? matchingResult.count : 0,
};
});
// Accumulate counts from top to bottom of the funnel
for (let i = filled.length - 1; i >= 0; i--) {
const step = filled[i];
const prevStep = filled[i + 1];
// If there's a previous step, add the count to the current step
if (step && prevStep) {
step.count += prevStep.count;
}
}
return filled.reverse();
}
export async function getFunnelData({
projectId,
startDate,
endDate,
...payload
}: IChartInput) {
const funnelWindow = (payload.funnelWindow || 24) * 3600;
const funnelGroup =
payload.funnelGroup === 'profile_id'
? [`COALESCE(nullIf(s.profile_id, ''), e.profile_id)`, 'profile_id']
: ['session_id', 'session_id'];
if (!startDate || !endDate) {
throw new Error('startDate and endDate are required');
}
if (payload.events.length === 0) {
return {
totalSessions: 0,
steps: [],
};
}
const funnels = payload.events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
const commonWhere = `project_id = ${sqlstring.escape(projectId)} AND
created_at >= '${formatClickhouseDate(startDate)}' AND
created_at <= '${formatClickhouseDate(endDate)}'`;
const innerSql = `SELECT
${funnelGroup[0]} AS ${funnelGroup[1]},
windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
FROM ${TABLE_NAMES.events} e
${funnelGroup[0] === 'session_id' ? '' : `LEFT JOIN (SELECT profile_id, id FROM sessions WHERE ${commonWhere}) AS s ON s.id = e.session_id`}
WHERE
${commonWhere} AND
name IN (${payload.events.map((event) => sqlstring.escape(event.name)).join(', ')})
GROUP BY ${funnelGroup[0]}`;
const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;
const funnel = await chQuery<{ level: number; count: number }>(sql);
const maxLevel = payload.events.length;
const filledFunnelRes = fillFunnel(funnel, maxLevel);
const totalSessions = last(filledFunnelRes)?.count ?? 0;
const steps = reverse(filledFunnelRes).reduce(
(acc, item, index, list) => {
const prev = list[index - 1] ?? { count: totalSessions };
const event = payload.events[item.level - 1]!;
return [
...acc,
{
event: {
...event,
displayName: event.displayName ?? event.name,
},
count: item.count,
percent: (item.count / totalSessions) * 100,
dropoffCount: prev.count - item.count,
dropoffPercent: 100 - (item.count / prev.count) * 100,
previousCount: prev.count,
},
];
},
[] as {
event: IChartEvent & { displayName: string };
count: number;
percent: number;
dropoffCount: number;
dropoffPercent: number;
previousCount: number;
}[],
);
return {
totalSessions,
steps,
};
}
export async function getChartSerie(
payload: IGetChartDataInput,
timezone: string,
) {
let result = await chQuery<ISerieDataItem>(
getChartSql({ ...payload, timezone }),
{
session_timezone: timezone,
},
);
if (result.length === 0 && payload.breakdowns.length > 0) {
result = await chQuery<ISerieDataItem>(
getChartSql({
...payload,
breakdowns: [],
timezone,
}),
{
session_timezone: timezone,
},
);
}
return groupByLabels(result).map((serie) => {
return {
...serie,
event: payload.event,
};
});
}
export type IGetChartSerie = Awaited<ReturnType<typeof getChartSeries>>[number];
export async function getChartSeries(
input: IChartInputWithDates,
timezone: string,
) {
const series = (
await Promise.all(
input.events.map(async (event) =>
getChartSerie(
{
...input,
event,
},
timezone,
),
),
)
).flat();
try {
return withFormula(input, series);
} catch (e) {
return series;
}
}
export async function getChart(input: IChartInput) {
const { timezone } = await getSettingsForProject(input.projectId);
const currentPeriod = getChartStartEndDate(input, timezone);
const previousPeriod = getChartPrevStartEndDate(currentPeriod);
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
currentPeriod.endDate,
);
if (endDate) {
currentPeriod.endDate = endDate;
}
const promises = [getChartSeries({ ...input, ...currentPeriod }, timezone)];
if (input.previous) {
promises.push(
getChartSeries(
{
...input,
...previousPeriod,
},
timezone,
),
);
}
const getSerieId = (serie: IGetChartSerie) =>
[slug(serie.name.join('-')), serie.event.id].filter(Boolean).join('-');
const result = await Promise.all(promises);
const series = result[0]!;
const previousSeries = result[1];
const limit = input.limit || 300;
const offset = input.offset || 0;
const includeEventAlphaId = input.events.length > 1;
const final: FinalChart = {
series: series.map((serie, index) => {
const eventIndex = input.events.findIndex(
(event) => event.id === serie.event.id,
);
const alphaId = alphabetIds[eventIndex];
const previousSerie = previousSeries?.find(
(prevSerie) => getSerieId(prevSerie) === getSerieId(serie),
);
const metrics = {
sum: sum(serie.data.map((item) => item.count)),
average: round(average(serie.data.map((item) => item.count)), 2),
min: min(serie.data.map((item) => item.count)),
max: max(serie.data.map((item) => item.count)),
count: serie.data[0]?.total_count, // We can grab any since all are the same
};
const event = {
id: serie.event.id,
name: serie.event.displayName || serie.event.name,
};
return {
id: getSerieId(serie),
names:
input.breakdowns.length === 0 && serie.event.displayName
? [serie.event.displayName]
: includeEventAlphaId
? [`(${alphaId}) ${serie.name[0]}`, ...serie.name.slice(1)]
: serie.name,
event,
metrics: {
...metrics,
...(input.previous
? {
previous: {
sum: getPreviousMetric(
metrics.sum,
previousSerie
? sum(previousSerie?.data.map((item) => item.count))
: null,
),
average: getPreviousMetric(
metrics.average,
previousSerie
? round(
average(
previousSerie?.data.map((item) => item.count),
),
2,
)
: null,
),
min: getPreviousMetric(
metrics.sum,
previousSerie
? min(previousSerie?.data.map((item) => item.count))
: null,
),
max: getPreviousMetric(
metrics.sum,
previousSerie
? max(previousSerie?.data.map((item) => item.count))
: null,
),
count: getPreviousMetric(
metrics.count ?? 0,
previousSerie?.data[0]?.total_count ?? null,
),
},
}
: {}),
},
data: serie.data.map((item, index) => ({
date: item.date,
count: item.count ?? 0,
previous: previousSerie?.data[index]
? getPreviousMetric(
item.count ?? 0,
previousSerie?.data[index]?.count ?? null,
)
: undefined,
})),
};
}),
metrics: {
sum: 0,
average: 0,
min: 0,
max: 0,
count: undefined,
},
};
// Sort by sum
final.series = final.series
.sort((a, b) => {
if (input.chartType === 'linear') {
const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
return sumB - sumA;
}
return (b.metrics[input.metric] ?? 0) - (a.metrics[input.metric] ?? 0);
})
.slice(offset, limit ? offset + limit : series.length);
final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
final.metrics.average = round(
average(final.series.map((item) => item.metrics.average)),
2,
);
final.metrics.min = min(final.series.map((item) => item.metrics.min));
final.metrics.max = max(final.series.map((item) => item.metrics.max));
if (input.previous) {
final.metrics.previous = {
sum: getPreviousMetric(
final.metrics.sum,
sum(final.series.map((item) => item.metrics.previous?.sum?.value ?? 0)),
),
average: getPreviousMetric(
final.metrics.average,
round(
average(
final.series.map(
(item) => item.metrics.previous?.average?.value ?? 0,
),
),
2,
),
),
min: getPreviousMetric(
final.metrics.min,
min(final.series.map((item) => item.metrics.previous?.min?.value ?? 0)),
),
max: getPreviousMetric(
final.metrics.max,
max(final.series.map((item) => item.metrics.previous?.max?.value ?? 0)),
),
count: undefined,
};
}
return final;
}

208
packages/trpc/src/routers/chart.ts
View File

@@ -10,22 +10,32 @@ import {
chQuery,
clix,
conversionService,
createSqlBuilder,
db,
formatClickhouseDate,
funnelService,
getChartPrevStartEndDate,
getChartStartEndDate,
getEventFiltersWhereClause,
getEventMetasCached,
getProfilesCached,
getSelectPropertyKey,
getSettingsForProject,
onlyReportEvents,
} from '@openpanel/db';
import {
type IChartEvent,
zChartEvent,
zChartEventFilter,
zChartInput,
zChartSeries,
zCriteria,
zRange,
zTimeInterval,
} from '@openpanel/validation';
import { round } from '@openpanel/common';
import { ChartEngine } from '@openpanel/db';
import {
differenceInDays,
differenceInMonths,
@@ -40,7 +50,6 @@ import {
protectedProcedure,
publicProcedure,
} from '../trpc';
import { getChart } from './chart.helpers';
function utc(date: string | Date) {
if (typeof date === 'string') {
@@ -402,7 +411,8 @@ export const chartRouter = createTRPCRouter({
}
}
return getChart(input);
// Use new chart engine
return ChartEngine.execute(input);
}),
cohort: protectedProcedure
.input(
@@ -532,6 +542,200 @@ export const chartRouter = createTRPCRouter({
return processCohortData(cohortData, diffInterval);
}),
getProfiles: protectedProcedure
.input(
z.object({
projectId: z.string(),
date: z.string().describe('The date for the data point (ISO string)'),
interval: zTimeInterval.default('day'),
series: zChartSeries,
breakdowns: z.record(z.string(), z.string()).optional(),
}),
)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const { projectId, date, series } = input;
const limit = 100;
const serie = series[0];
if (!serie) {
throw new Error('Series not found');
}
if (serie.type !== 'event') {
throw new Error('Series must be an event');
}
// Build the date range for the specific interval bucket
const dateObj = new Date(date);
// Build query to get unique profile_ids for this time bucket
const { sb, getSql } = createSqlBuilder();
sb.select.profile_id = 'DISTINCT profile_id';
sb.where = getEventFiltersWhereClause(serie.filters);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
sb.where.dateRange = `${clix.toStartOf('created_at', input.interval)} = ${clix.toDate(sqlstring.escape(formatClickhouseDate(dateObj)), input.interval)}`;
if (serie.name !== '*') {
sb.where.eventName = `name = ${sqlstring.escape(serie.name)}`;
}
console.log('> breakdowns', input.breakdowns);
if (input.breakdowns) {
Object.entries(input.breakdowns).forEach(([key, value]) => {
sb.where[`breakdown_${key}`] = `${key} = ${sqlstring.escape(value)}`;
});
}
// // Handle breakdowns if provided
// const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
// breakdown.name.startsWith('profile.'),
// );
// const anyFilterOnProfile = [...event.filters, ...filters].some((filter) =>
// filter.name.startsWith('profile.'),
// );
// if (anyFilterOnProfile || anyBreakdownOnProfile) {
// sb.joins.profiles = `LEFT ANY JOIN (SELECT
// id as "profile.id",
// email as "profile.email",
// first_name as "profile.first_name",
// last_name as "profile.last_name",
// properties as "profile.properties"
// FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
// }
// Apply breakdown filters if provided
// breakdowns.forEach((breakdown) => {
// // This is simplified - in reality we'd need to match the breakdown value
// // For now, we'll just get all profiles for the time bucket
// });
// Get unique profile IDs
const profileIds = await chQuery<{ profile_id: string }>(getSql());
if (profileIds.length === 0) {
return [];
}
// Fetch profile details
const ids = profileIds.map((p) => p.profile_id).filter(Boolean);
const profiles = await getProfilesCached(ids, projectId);
return profiles;
}),
getFunnelProfiles: protectedProcedure
.input(
z.object({
projectId: z.string(),
startDate: z.string().nullish(),
endDate: z.string().nullish(),
series: zChartSeries,
stepIndex: z.number().describe('0-based index of the funnel step'),
showDropoffs: z
.boolean()
.optional()
.default(false)
.describe(
'If true, show users who dropped off at this step. If false, show users who completed at least this step.',
),
funnelWindow: z.number().optional(),
funnelGroup: z.string().optional(),
breakdowns: z.array(z.object({ name: z.string() })).optional(),
range: zRange,
}),
)
.query(async ({ input }) => {
const { timezone } = await getSettingsForProject(input.projectId);
const {
projectId,
series,
stepIndex,
showDropoffs = false,
funnelWindow,
funnelGroup,
breakdowns = [],
} = input;
const { startDate, endDate } = getChartStartEndDate(input, timezone);
// stepIndex is 0-based, but level is 1-based, so we need level >= stepIndex + 1
const targetLevel = stepIndex + 1;
const eventSeries = onlyReportEvents(series);
if (eventSeries.length === 0) {
throw new Error('At least one event series is required');
}
const funnelWindowSeconds = (funnelWindow || 24) * 3600;
const funnelWindowMilliseconds = funnelWindowSeconds * 1000;
// Use funnel service methods
const group = funnelService.getFunnelGroup(funnelGroup);
// Create sessions CTE if needed
const sessionsCte =
group[0] !== 'session_id'
? funnelService.buildSessionsCte({
projectId,
startDate,
endDate,
timezone,
})
: null;
// Create funnel CTE using funnel service
const funnelCte = funnelService.buildFunnelCte({
projectId,
startDate,
endDate,
eventSeries: eventSeries as IChartEvent[],
funnelWindowMilliseconds,
group,
timezone,
additionalSelects: ['profile_id'],
additionalGroupBy: ['profile_id'],
});
// Build main query
const query = clix(ch, timezone);
if (sessionsCte) {
funnelCte.leftJoin('sessions s', 's.sid = events.session_id');
query.with('sessions', sessionsCte);
}
query.with('funnel', funnelCte);
// Get distinct profile IDs
query
.select(['DISTINCT profile_id'])
.from('funnel')
.where('level', '!=', 0);
if (showDropoffs) {
// Show users who dropped off at this step (completed this step but not the next)
query.where('level', '=', targetLevel);
} else {
// Show users who completed at least this step
query.where('level', '>=', targetLevel);
}
const profileIdsResult = (await query.execute()) as {
profile_id: string;
}[];
if (profileIdsResult.length === 0) {
return [];
}
// Fetch profile details
const ids = profileIdsResult.map((p) => p.profile_id).filter(Boolean);
const profiles = await getProfilesCached(ids, projectId);
return profiles;
}),
});
function processCohortData(

4
packages/trpc/src/routers/report.ts
View File

@@ -46,7 +46,7 @@ export const reportRouter = createTRPCRouter({
projectId: dashboard.projectId,
dashboardId,
name: report.name,
events: report.events,
events: report.series,
interval: report.interval,
breakdowns: report.breakdowns,
chartType: report.chartType,
@@ -91,7 +91,7 @@ export const reportRouter = createTRPCRouter({
},
data: {
name: report.name,
events: report.events,
events: report.series,
interval: report.interval,
breakdowns: report.breakdowns,
chartType: report.chartType,

76
packages/validation/src/index.ts
View File

@@ -57,12 +57,70 @@ export const zChartEvent = z.object({
.default([])
.describe('Filters applied specifically to this event'),
});
export const zChartFormula = z.object({
id: z
.string()
.optional()
.describe('Unique identifier for the formula configuration'),
type: z.literal('formula'),
formula: z.string().describe('The formula expression (e.g., A+B, A/B)'),
displayName: z
.string()
.optional()
.describe('A user-friendly name for display purposes'),
});
// Event with type field for discriminated union
export const zChartEventWithType = zChartEvent.extend({
type: z.literal('event'),
});
export const zChartEventItem = z.discriminatedUnion('type', [
zChartEventWithType,
zChartFormula,
]);
export const zChartBreakdown = z.object({
id: z.string().optional(),
name: z.string(),
});
export const zChartEvents = z.array(zChartEvent);
// Support both old format (array of events without type) and new format (array of event/formula items)
// Preprocess to normalize: if item has 'type' field, use discriminated union; otherwise, add type: 'event'
export const zChartSeries = z.preprocess((val) => {
if (!val) return val;
let processedVal = val;
// If the input is an object with numeric keys, convert it to an array
if (typeof val === 'object' && val !== null && !Array.isArray(val)) {
const keys = Object.keys(val).sort(
(a, b) => Number.parseInt(a) - Number.parseInt(b),
);
processedVal = keys.map((key) => (val as any)[key]);
}
if (!Array.isArray(processedVal)) return processedVal;
return processedVal.map((item: any) => {
// If item already has type field, return as-is
if (item && typeof item === 'object' && 'type' in item) {
return item;
}
// Otherwise, add type: 'event' for backward compatibility
if (item && typeof item === 'object' && 'name' in item) {
return { ...item, type: 'event' };
}
return item;
});
}, z
.array(zChartEventItem)
.describe(
'Array of series (events or formulas) to be tracked and displayed in the chart',
));
// Keep zChartEvents as an alias for backward compatibility during migration
export const zChartEvents = zChartSeries;
export const zChartBreakdowns = z.array(zChartBreakdown);
export const zChartType = z.enum(objectToZodEnums(chartTypes));
@@ -77,7 +135,7 @@ export const zRange = z.enum(objectToZodEnums(timeWindows));
export const zCriteria = z.enum(['on_or_after', 'on']);
export const zChartInput = z.object({
export const zChartInputBase = z.object({
chartType: zChartType
.default('linear')
.describe('What type of chart should be displayed'),
@@ -86,8 +144,8 @@ export const zChartInput = z.object({
.describe(
'The time interval for data aggregation (e.g., day, week, month)',
),
events: zChartEvents.describe(
'Array of events to be tracked and displayed in the chart',
series: zChartSeries.describe(
'Array of series (events or formulas) to be tracked and displayed in the chart',
),
breakdowns: zChartBreakdowns
.default([])
@@ -144,7 +202,15 @@ export const zChartInput = z.object({
.describe('Time window in hours for funnel analysis'),
});
export const zReportInput = zChartInput.extend({
export const zChartInput = z.preprocess((val) => {
if (val && typeof val === 'object' && 'events' in val && !('series' in val)) {
// Migrate old 'events' field to 'series'
return { ...val, series: val.events };
}
return val;
}, zChartInputBase);
export const zReportInput = zChartInputBase.extend({
name: z.string().describe('The user-defined name for the report'),
lineType: zLineType.describe('The visual style of the line in the chart'),
unit: z

28
packages/validation/src/test.ts Normal file
View File

@@ -0,0 +1,28 @@
import { zChartEvents } from '.';
const events = [
{
id: 'sAmT',
type: 'event',
name: 'session_end',
segment: 'event',
filters: [],
},
{
id: '5K2v',
type: 'event',
name: 'session_start',
segment: 'event',
filters: [],
},
{
id: 'lQiQ',
type: 'formula',
formula: 'A/B',
displayName: '',
},
];
const res = zChartEvents.safeParse(events);
console.log(res);

15
packages/validation/src/types.validation.ts
View File

@@ -1,11 +1,18 @@
import type { z } from 'zod';
export type UnionOmit<T, K extends keyof any> = T extends any
? Omit<T, K>
: never;
import type {
zChartBreakdown,
zChartEvent,
zChartEventItem,
zChartEventSegment,
zChartFormula,
zChartInput,
zChartInputAI,
zChartSeries,
zChartType,
zCriteria,
zLineType,
@@ -24,6 +31,11 @@ export type IChartProps = z.infer<typeof zReportInput> & {
previousIndicatorInverted?: boolean;
};
export type IChartEvent = z.infer<typeof zChartEvent>;
export type IChartFormula = z.infer<typeof zChartFormula>;
export type IChartEventItem = z.infer<typeof zChartEventItem>;
export type IChartSeries = z.infer<typeof zChartSeries>;
// Backward compatibility alias
export type IChartEvents = IChartSeries;
export type IChartEventSegment = z.infer<typeof zChartEventSegment>;
export type IChartEventFilter = IChartEvent['filters'][number];
export type IChartEventFilterValue =
@@ -45,7 +57,7 @@ export type IGetChartDataInput = {
projectId: string;
startDate: string;
endDate: string;
} & Omit<IChartInput, 'events' | 'name' | 'startDate' | 'endDate' | 'range'>;
} & Omit<IChartInput, 'series' | 'name' | 'startDate' | 'endDate' | 'range'>;
export type ICriteria = z.infer<typeof zCriteria>;
export type PreviousValue =
@@ -77,6 +89,7 @@ export type IChartSerie = {
event: {
id?: string;
name: string;
breakdowns?: Record<string, string>;
};
metrics: Metrics;
data: {