zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: report editor

Browse Source 
commit bfcf271a64c33a60f61f511cec2198d9c8a9c51a
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Wed Nov 26 12:32:40 2025 +0100

    wip

commit 8cd3b89fa3
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 22:33:58 2025 +0100

    funnel

commit 95af86dc44
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 22:23:25 2025 +0100

    wip

commit 727a218e6b
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 10:18:26 2025 +0100

    conversion wip

commit 958ba535d6
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 10:18:20 2025 +0100

    wip

commit 3bbeb927cc
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Tue Nov 25 09:18:48 2025 +0100

    wip

commit d99335e2f4
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 18:08:10 2025 +0100

    wip

commit 1fa61b1ae9
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 15:50:28 2025 +0100

    ts

commit 548747d826
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 13:17:01 2025 +0100

    fix typecheck events -> series

commit 7b18544085
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Mon Nov 24 13:06:46 2025 +0100

    fix report table

commit 57697a5a39
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Sat Nov 22 00:05:13 2025 +0100

    wip

commit 06fb6c4f3c
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Fri Nov 21 11:21:17 2025 +0100

    wip

commit dd71fd4e11
Author: Carl-Gerhard Lindesvärd <lindesvard@gmail.com>
Date:   Thu Nov 20 13:56:58 2025 +0100

    formulas
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-26 12:33:41 +01:00
parent 828c8c4f91
commit b421474616
70 changed files with 6867 additions and 1918 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
packages/db/src/services/chart.service.ts
View File

@@ -155,7 +155,8 @@ export function getChartSql({
}
breakdowns.forEach((breakdown, index) => {
const key = `label_${index}`;
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -175,8 +176,8 @@ export function getChartSql({
if (event.segment === 'property_sum' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `sum(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'sum(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `sum(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -185,8 +186,8 @@ export function getChartSql({
if (event.segment === 'property_average' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `avg(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'avg(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `avg(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -195,8 +196,8 @@ export function getChartSql({
if (event.segment === 'property_max' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `max(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'max(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `max(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -205,8 +206,8 @@ export function getChartSql({
if (event.segment === 'property_min' && event.property) {
if (event.property === 'revenue') {
sb.select.count = `min(revenue) as count`;
sb.where.property = `revenue > 0`;
sb.select.count = 'min(revenue) as count';
sb.where.property = 'revenue > 0';
} else {
sb.select.count = `min(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
@@ -230,14 +231,58 @@ export function getChartSql({
return sql;
}
// Add total unique count for user segment using a scalar subquery
if (event.segment === 'user') {
const totalUniqueSubquery = `(
SELECT ${sb.select.count}
// Build total_count calculation that accounts for breakdowns
// When breakdowns exist, we need to calculate total_count per breakdown group
if (breakdowns.length > 0) {
// Create a subquery that calculates total_count per breakdown group (without date grouping)
// Then reference it in the main query via JOIN
const breakdownSelects = breakdowns
.map((breakdown, index) => {
const key = `label_${index + 1}`;
const breakdownExpr = getSelectPropertyKey(breakdown.name);
return `${breakdownExpr} as ${key}`;
})
.join(', ');
// GROUP BY needs to use the actual expressions, not aliases
const breakdownGroupByExprs = breakdowns
.map((breakdown) => getSelectPropertyKey(breakdown.name))
.join(', ');
// Build the total_count subquery grouped only by breakdowns (no date)
// Extract the count expression without the alias (remove "as count")
const countExpression = sb.select.count.replace(/\s+as\s+count$/i, '');
const totalCountSubquery = `(
SELECT
${breakdownSelects},
${countExpression} as total_count
FROM ${sb.from}
${getJoins()}
${getWhere()}
)`;
GROUP BY ${breakdownGroupByExprs}
) as total_counts`;
// Join the total_counts subquery to get total_count per breakdown
// Match on the breakdown column values
const joinConditions = breakdowns
.map((_, index) => {
const outerKey = `label_${index + 1}`;
return `${outerKey} = total_counts.label_${index + 1}`;
})
.join(' AND ');
sb.joins.total_counts = `LEFT JOIN ${totalCountSubquery} ON ${joinConditions}`;
// Use any() aggregate since total_count is the same for all rows in a breakdown group
sb.select.total_unique_count =
'any(total_counts.total_count) as total_count';
} else {
// No breakdowns - use a simple subquery for total count
const totalUniqueSubquery = `(
SELECT ${sb.select.count}
FROM ${sb.from}
${getJoins()}
${getWhere()}
)`;
sb.select.total_unique_count = `${totalUniqueSubquery} as total_count`;
}
@@ -509,12 +554,11 @@ export function getChartStartEndDate(
}: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>,
timezone: string,
) {
const ranges = getDatesFromRange(range, timezone);
if (startDate && endDate) {
return { startDate: startDate, endDate: endDate };
}
const ranges = getDatesFromRange(range, timezone);
if (!startDate && endDate) {
return { startDate: ranges.startDate, endDate: endDate };
}

39
packages/db/src/services/conversion.service.ts
View File

@@ -1,5 +1,5 @@
import { NOT_SET_VALUE } from '@openpanel/constants';
import type { IChartInput } from '@openpanel/validation';
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { omit } from 'ramda';
import { TABLE_NAMES, ch } from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
@@ -7,6 +7,7 @@ import {
getEventFiltersWhereClause,
getSelectPropertyKey,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
export class ConversionService {
constructor(private client: typeof ch) {}
@@ -17,8 +18,9 @@ export class ConversionService {
endDate,
funnelGroup,
funnelWindow = 24,
events,
series,
breakdowns = [],
limit,
interval,
timezone,
}: Omit<IChartInput, 'range' | 'previous' | 'metric' | 'chartType'> & {
@@ -30,6 +32,8 @@ export class ConversionService {
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
const events = onlyReportEvents(series);
if (events.length !== 2) {
throw new Error('events must be an array of two events');
}
@@ -111,18 +115,20 @@ export class ConversionService {
}
const results = await query.execute();
return this.toSeries(results, breakdowns).map((serie, serieIndex) => {
return {
...serie,
data: serie.data.map((d, index) => ({
...d,
timestamp: new Date(d.date).getTime(),
serieIndex,
index,
serie: omit(['data'], serie),
})),
};
});
return this.toSeries(results, breakdowns, limit).map(
(serie, serieIndex) => {
return {
...serie,
data: serie.data.map((d, index) => ({
...d,
timestamp: new Date(d.date).getTime(),
serieIndex,
index,
serie: omit(['data'], serie),
})),
};
},
);
}
private toSeries(
@@ -134,6 +140,7 @@ export class ConversionService {
[key: string]: string | number;
}[],
breakdowns: { name: string }[] = [],
limit: number | undefined = undefined,
) {
if (!breakdowns.length) {
return [
@@ -153,6 +160,10 @@ export class ConversionService {
// Group by breakdown values
const series = data.reduce(
(acc, d) => {
if (limit && Object.keys(acc).length >= limit) {
return acc;
}
const key =
breakdowns.map((b, index) => d[`b_${index}`]).join('|') ||
NOT_SET_VALUE;

149
packages/db/src/services/funnel.service.ts
View File

@@ -1,5 +1,9 @@
import { ifNaN } from '@openpanel/common';
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import type {
IChartEvent,
IChartEventItem,
IChartInput,
} from '@openpanel/validation';
import { last, reverse, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { ch } from '../clickhouse/client';
@@ -10,17 +14,18 @@ import {
getEventFiltersWhereClause,
getSelectPropertyKey,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
export class FunnelService {
constructor(private client: typeof ch) {}
private getFunnelGroup(group?: string) {
getFunnelGroup(group?: string): [string, string] {
return group === 'profile_id'
? [`COALESCE(nullIf(s.pid, ''), profile_id)`, 'profile_id']
: ['session_id', 'session_id'];
}
private getFunnelConditions(events: IChartEvent[]) {
getFunnelConditions(events: IChartEvent[] = []): string[] {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
@@ -29,6 +34,70 @@ export class FunnelService {
});
}
buildFunnelCte({
projectId,
startDate,
endDate,
eventSeries,
funnelWindowMilliseconds,
group,
timezone,
additionalSelects = [],
additionalGroupBy = [],
}: {
projectId: string;
startDate: string;
endDate: string;
eventSeries: IChartEvent[];
funnelWindowMilliseconds: number;
group: [string, string];
timezone: string;
additionalSelects?: string[];
additionalGroupBy?: string[];
}) {
const funnels = this.getFunnelConditions(eventSeries);
return clix(this.client, timezone)
.select([
`${group[0]} AS ${group[1]}`,
...additionalSelects,
`windowFunnel(${funnelWindowMilliseconds}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), ${funnels.join(', ')}) AS level`,
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.where(
'name',
'IN',
eventSeries.map((e) => e.name),
)
.groupBy([group[1], ...additionalGroupBy]);
}
buildSessionsCte({
projectId,
startDate,
endDate,
timezone,
}: {
projectId: string;
startDate: string;
endDate: string;
timezone: string;
}) {
return clix(this.client, timezone)
.select(['profile_id as pid', 'id as sid'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
]);
}
private fillFunnel(
funnel: { level: number; count: number }[],
steps: number,
@@ -57,6 +126,7 @@ export class FunnelService {
toSeries(
funnel: { level: number; count: number; [key: string]: any }[],
breakdowns: { name: string }[] = [],
limit: number | undefined = undefined,
) {
if (!breakdowns.length) {
return [
@@ -72,6 +142,10 @@ export class FunnelService {
// Group by breakdown values
const series = funnel.reduce(
(acc, f) => {
if (limit && Object.keys(acc).length >= limit) {
return acc;
}
const key = breakdowns.map((b, index) => f[`b_${index}`]).join('|');
if (!acc[key]) {
acc[key] = [];
@@ -110,51 +184,49 @@ export class FunnelService {
projectId,
startDate,
endDate,
events,
series,
funnelWindow = 24,
funnelGroup,
breakdowns = [],
limit,
timezone = 'UTC',
}: IChartInput & { timezone: string }) {
}: IChartInput & { timezone: string; events?: IChartEvent[] }) {
if (!startDate || !endDate) {
throw new Error('startDate and endDate are required');
}
if (events.length === 0) {
const eventSeries = onlyReportEvents(series);
if (eventSeries.length === 0) {
throw new Error('events are required');
}
const funnelWindowSeconds = funnelWindow * 3600;
const funnelWindowMilliseconds = funnelWindowSeconds * 1000;
const group = this.getFunnelGroup(funnelGroup);
const funnels = this.getFunnelConditions(events);
const profileFilters = this.getProfileFilters(events);
const profileFilters = this.getProfileFilters(eventSeries);
const anyFilterOnProfile = profileFilters.length > 0;
const anyBreakdownOnProfile = breakdowns.some((b) =>
b.name.startsWith('profile.'),
);
// Create the funnel CTE
const funnelCte = clix(this.client, timezone)
.select([
`${group[0]} AS ${group[1]}`,
...breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
),
`windowFunnel(${funnelWindowMilliseconds}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), ${funnels.join(', ')}) AS level`,
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.where(
'name',
'IN',
events.map((e) => e.name),
)
.groupBy([group[1], ...breakdowns.map((b, index) => `b_${index}`)]);
const breakdownSelects = breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
const funnelCte = this.buildFunnelCte({
projectId,
startDate,
endDate,
eventSeries,
funnelWindowMilliseconds,
group,
timezone,
additionalSelects: breakdownSelects,
additionalGroupBy: breakdownGroupBy,
});
if (anyFilterOnProfile || anyBreakdownOnProfile) {
funnelCte.leftJoin(
@@ -167,15 +239,12 @@ export class FunnelService {
// Create the sessions CTE if needed
const sessionsCte =
group[0] !== 'session_id'
? clix(this.client, timezone)
// Important to have unique field names to avoid ambiguity in the main query
.select(['profile_id as pid', 'id as sid'])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
? this.buildSessionsCte({
projectId,
startDate,
endDate,
timezone,
})
: null;
// Base funnel query with CTEs
@@ -204,11 +273,11 @@ export class FunnelService {
.orderBy('level', 'DESC');
const funnelData = await funnelQuery.execute();
const funnelSeries = this.toSeries(funnelData, breakdowns);
const funnelSeries = this.toSeries(funnelData, breakdowns, limit);
return funnelSeries
.map((data) => {
const maxLevel = events.length;
const maxLevel = eventSeries.length;
const filledFunnelRes = this.fillFunnel(
data.map((d) => ({ level: d.level, count: d.count })),
maxLevel,
@@ -220,7 +289,7 @@ export class FunnelService {
(acc, item, index, list) => {
const prev = list[index - 1] ?? { count: totalSessions };
const next = list[index + 1];
const event = events[item.level - 1]!;
const event = eventSeries[item.level - 1]!;
return [
...acc,
{

43
packages/db/src/services/reports.service.ts
View File

@@ -5,19 +5,25 @@ import {
} from '@openpanel/constants';
import type {
IChartBreakdown,
IChartEvent,
IChartEventFilter,
IChartEventItem,
IChartLineType,
IChartProps,
IChartRange,
ICriteria,
} from '@openpanel/validation';
import { db } from '../prisma-client';
import type { Report as DbReport, ReportLayout } from '../prisma-client';
import { db } from '../prisma-client';
export type IServiceReport = Awaited<ReturnType<typeof getReportById>>;
export const onlyReportEvents = (
series: NonNullable<IServiceReport>['series'],
) => {
return series.filter((item) => item.type === 'event');
};
export function transformFilter(
filter: Partial<IChartEventFilter>,
index: number,
@@ -31,17 +37,29 @@ export function transformFilter(
};
}
export function transformReportEvent(
event: Partial<IChartEvent>,
export function transformReportEventItem(
item: IChartEventItem,
index: number,
): IChartEvent {
): IChartEventItem {
if (item.type === 'formula') {
// Transform formula
return {
type: 'formula',
id: item.id ?? alphabetIds[index]!,
formula: item.formula || '',
displayName: item.displayName,
};
}
// Transform event with type field
return {
segment: event.segment ?? 'event',
filters: (event.filters ?? []).map(transformFilter),
id: event.id ?? alphabetIds[index]!,
name: event.name || 'unknown_event',
displayName: event.displayName,
property: event.property,
type: 'event',
segment: item.segment ?? 'event',
filters: (item.filters ?? []).map(transformFilter),
id: item.id ?? alphabetIds[index]!,
name: item.name || 'unknown_event',
displayName: item.displayName,
property: item.property,
};
}
@@ -51,7 +69,8 @@ export function transformReport(
return {
id: report.id,
projectId: report.projectId,
events: (report.events as IChartEvent[]).map(transformReportEvent),
series:
(report.events as IChartEventItem[]).map(transformReportEventItem) ?? [],
breakdowns: report.breakdowns as IChartBreakdown[],
chartType: report.chartType,
lineType: (report.lineType as IChartLineType) ?? lineTypes.monotone,