fix: dashboard improvements and query speed improvements

This commit is contained in:
Carl-Gerhard Lindesvärd
2026-01-09 14:42:11 +01:00
parent 4867260ece
commit cabfb1f3f0
49 changed files with 3398 additions and 950 deletions

View File

@@ -28,5 +28,6 @@ export * from './src/types';
export * from './src/clickhouse/query-builder';
export * from './src/services/import.service';
export * from './src/services/overview.service';
export * from './src/services/pages.service';
export * from './src/services/insights';
export * from './src/session-context';

View File

@@ -90,6 +90,7 @@ function getClickhouseSettings(): ClickHouseSettings {
{};
return {
distributed_product_mode: 'allow',
date_time_input_format: 'best_effort',
...(!process.env.CLICKHOUSE_SETTINGS_REMOVE_CONVERT_ANY_JOIN
? {

View File

@@ -519,7 +519,7 @@ export class Query<T = any> {
const query = this.buildQuery();
console.log(
'query',
`${query} SETTINGS session_timezone = '${this.timezone}'`,
`${query.replaceAll('\n', ' ').replaceAll('\t', ' ').replaceAll('\r', ' ')} SETTINGS session_timezone = '${this.timezone}'`,
);
const result = await this.client.query({

View File

@@ -1,7 +1,16 @@
import { getPreviousMetric } from '@openpanel/common';
import type { FinalChart, IChartInput } from '@openpanel/validation';
import { getChartPrevStartEndDate } from '../services/chart.service';
import { getPreviousMetric, groupByLabels } from '@openpanel/common';
import type { ISerieDataItem } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import type {
FinalChart,
IChartEventItem,
IChartInput,
} from '@openpanel/validation';
import { chQuery } from '../clickhouse/client';
import {
getAggregateChartSql,
getChartPrevStartEndDate,
} from '../services/chart.service';
import {
getOrganizationSubscriptionChartEndDate,
getSettingsForProject,
@@ -69,7 +78,280 @@ export async function executeChart(input: IChartInput): Promise<FinalChart> {
return response;
}
/**
* Aggregate Chart Engine - Optimized for bar/pie charts without time series
* Executes a simplified pipeline: normalize -> fetch aggregate -> format
*/
export async function executeAggregateChart(
input: IChartInput,
): Promise<FinalChart> {
// Stage 1: Normalize input
const normalized = await normalize(input);
// Handle subscription end date limit
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
normalized.endDate,
);
if (endDate) {
normalized.endDate = endDate;
}
const { timezone } = await getSettingsForProject(normalized.projectId);
// Stage 2: Fetch aggregate data for current period (event series only)
const fetchedSeries: ConcreteSeries[] = [];
for (let i = 0; i < normalized.series.length; i++) {
const definition = normalized.series[i]!;
if (definition.type !== 'event') {
// Skip formulas - they'll be computed in the next stage
continue;
}
const event = definition as IChartEventItem & { type: 'event' };
// Build query input
const queryInput = {
event: {
id: event.id,
name: event.name,
segment: event.segment,
filters: event.filters,
displayName: event.displayName,
property: event.property,
},
projectId: normalized.projectId,
startDate: normalized.startDate,
endDate: normalized.endDate,
breakdowns: normalized.breakdowns,
limit: normalized.limit,
timezone,
};
// Execute aggregate query
let queryResult = await chQuery<ISerieDataItem>(
getAggregateChartSql(queryInput),
{
session_timezone: timezone,
},
);
// Fallback: if no results with breakdowns, try without breakdowns
if (queryResult.length === 0 && normalized.breakdowns.length > 0) {
queryResult = await chQuery<ISerieDataItem>(
getAggregateChartSql({
...queryInput,
breakdowns: [],
}),
{
session_timezone: timezone,
},
);
}
// Group by labels (handles breakdown expansion)
const groupedSeries = groupByLabels(queryResult);
// Create concrete series for each grouped result
groupedSeries.forEach((grouped) => {
// Extract breakdown value from name array
const breakdownValue =
normalized.breakdowns.length > 0 && grouped.name.length > 1
? grouped.name.slice(1).join(' - ')
: undefined;
// Build breakdowns object
const breakdowns: Record<string, string> | undefined =
normalized.breakdowns.length > 0 && grouped.name.length > 1
? {}
: undefined;
if (breakdowns) {
normalized.breakdowns.forEach((breakdown, idx) => {
const breakdownNamePart = grouped.name[idx + 1];
if (breakdownNamePart) {
breakdowns[breakdown.name] = breakdownNamePart;
}
});
}
// Build filters including breakdown value
const filters = [...event.filters];
if (breakdownValue && normalized.breakdowns.length > 0) {
normalized.breakdowns.forEach((breakdown, idx) => {
const breakdownNamePart = grouped.name[idx + 1];
if (breakdownNamePart) {
filters.push({
id: `breakdown-${idx}`,
name: breakdown.name,
operator: 'is',
value: [breakdownNamePart],
});
}
});
}
// For aggregate charts, grouped.data should have a single data point
// (since we use a constant date in the query)
const concrete: ConcreteSeries = {
id: `${event.name}-${grouped.name.join('-')}-${i}`,
definitionId: definition.id ?? alphabetIds[i] ?? `series-${i}`,
definitionIndex: i,
name: grouped.name,
context: {
event: event.name,
filters,
breakdownValue,
breakdowns,
},
data: grouped.data,
definition,
};
fetchedSeries.push(concrete);
});
}
// Stage 3: Compute formula series from fetched event series
const computedSeries = compute(fetchedSeries, normalized.series);
// Stage 4: Fetch previous period if requested
let previousSeries: ConcreteSeries[] | null = null;
if (input.previous) {
const currentPeriod = {
startDate: normalized.startDate,
endDate: normalized.endDate,
};
const previousPeriod = getChartPrevStartEndDate(currentPeriod);
const previousFetchedSeries: ConcreteSeries[] = [];
for (let i = 0; i < normalized.series.length; i++) {
const definition = normalized.series[i]!;
if (definition.type !== 'event') {
continue;
}
const event = definition as IChartEventItem & { type: 'event' };
const queryInput = {
event: {
id: event.id,
name: event.name,
segment: event.segment,
filters: event.filters,
displayName: event.displayName,
property: event.property,
},
projectId: normalized.projectId,
startDate: previousPeriod.startDate,
endDate: previousPeriod.endDate,
breakdowns: normalized.breakdowns,
limit: normalized.limit,
timezone,
};
let queryResult = await chQuery<ISerieDataItem>(
getAggregateChartSql(queryInput),
{
session_timezone: timezone,
},
);
if (queryResult.length === 0 && normalized.breakdowns.length > 0) {
queryResult = await chQuery<ISerieDataItem>(
getAggregateChartSql({
...queryInput,
breakdowns: [],
}),
{
session_timezone: timezone,
},
);
}
const groupedSeries = groupByLabels(queryResult);
groupedSeries.forEach((grouped) => {
const breakdownValue =
normalized.breakdowns.length > 0 && grouped.name.length > 1
? grouped.name.slice(1).join(' - ')
: undefined;
const breakdowns: Record<string, string> | undefined =
normalized.breakdowns.length > 0 && grouped.name.length > 1
? {}
: undefined;
if (breakdowns) {
normalized.breakdowns.forEach((breakdown, idx) => {
const breakdownNamePart = grouped.name[idx + 1];
if (breakdownNamePart) {
breakdowns[breakdown.name] = breakdownNamePart;
}
});
}
const filters = [...event.filters];
if (breakdownValue && normalized.breakdowns.length > 0) {
normalized.breakdowns.forEach((breakdown, idx) => {
const breakdownNamePart = grouped.name[idx + 1];
if (breakdownNamePart) {
filters.push({
id: `breakdown-${idx}`,
name: breakdown.name,
operator: 'is',
value: [breakdownNamePart],
});
}
});
}
const concrete: ConcreteSeries = {
id: `${event.name}-${grouped.name.join('-')}-${i}`,
definitionId: definition.id ?? alphabetIds[i] ?? `series-${i}`,
definitionIndex: i,
name: grouped.name,
context: {
event: event.name,
filters,
breakdownValue,
breakdowns,
},
data: grouped.data,
definition,
};
previousFetchedSeries.push(concrete);
});
}
// Compute formula series for previous period
previousSeries = compute(previousFetchedSeries, normalized.series);
}
// Stage 5: Format final output with previous period data
const includeAlphaIds = normalized.series.length > 1;
const response = format(
computedSeries,
normalized.series,
includeAlphaIds,
previousSeries,
normalized.limit,
);
return response;
}
// Export as ChartEngine for backward compatibility
export const ChartEngine = {
execute: executeChart,
};
// Export aggregate chart engine
export const AggregateChartEngine = {
execute: executeAggregateChart,
};

View File

@@ -348,6 +348,246 @@ export function getChartSql({
return sql;
}
export function getAggregateChartSql({
event,
breakdowns,
startDate,
endDate,
projectId,
limit,
timezone,
}: Omit<IGetChartDataInput, 'interval' | 'chartType'> & {
timezone: string;
}) {
const {
sb,
join,
getWhere,
getFrom,
getJoins,
getSelect,
getOrderBy,
getGroupBy,
getWith,
with: addCte,
getSql,
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
filter.name.startsWith('profile.'),
);
const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
breakdown.name.startsWith('profile.'),
);
// Build WHERE clause without the bar filter (for use in subqueries and CTEs)
const getWhereWithoutBar = () => {
const whereWithoutBar = { ...sb.where };
delete whereWithoutBar.bar;
return Object.keys(whereWithoutBar).length
? `WHERE ${join(whereWithoutBar, ' AND ')}`
: '';
};
// Collect all profile fields used in filters and breakdowns
const getProfileFields = () => {
const fields = new Set<string>();
// Always need id for the join
fields.add('id');
// Collect from filters
event.filters
.filter((f) => f.name.startsWith('profile.'))
.forEach((f) => {
const fieldName = f.name.replace('profile.', '').split('.')[0];
if (fieldName && fieldName === 'properties') {
fields.add('properties');
} else if (
fieldName &&
['email', 'first_name', 'last_name'].includes(fieldName)
) {
fields.add(fieldName);
}
});
// Collect from breakdowns
breakdowns
.filter((b) => b.name.startsWith('profile.'))
.forEach((b) => {
const fieldName = b.name.replace('profile.', '').split('.')[0];
if (fieldName && fieldName === 'properties') {
fields.add('properties');
} else if (
fieldName &&
['email', 'first_name', 'last_name'].includes(fieldName)
) {
fields.add(fieldName);
}
});
return Array.from(fields);
};
// Create profiles CTE if profiles are needed
const profilesJoinRef =
anyFilterOnProfile || anyBreakdownOnProfile
? 'LEFT ANY JOIN profile ON profile.id = profile_id'
: '';
if (anyFilterOnProfile || anyBreakdownOnProfile) {
const profileFields = getProfileFields();
const selectFields = profileFields.map((field) => {
if (field === 'id') {
return 'id as "profile.id"';
}
if (field === 'properties') {
return 'properties as "profile.properties"';
}
if (field === 'email') {
return 'email as "profile.email"';
}
if (field === 'first_name') {
return 'first_name as "profile.first_name"';
}
if (field === 'last_name') {
return 'last_name as "profile.last_name"';
}
return field;
});
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
);
sb.joins.profiles = profilesJoinRef;
}
// Date range filters
if (startDate) {
sb.where.startDate = `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`;
}
if (endDate) {
sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
}
// Add a constant date field for aggregate charts (groupByLabels expects it)
// Use startDate as the date value since we're aggregating across the entire range
sb.select.date = `${sqlstring.escape(startDate)} as date`;
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
.map((b) => getSelectPropertyKey(b.name))
.join(', ');
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
// Add breakdowns to SELECT and GROUP BY
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
// Always group by label_0 (event name) for aggregate charts
sb.groupBy.label_0 = 'label_0';
// Default count aggregation
sb.select.count = 'count(*) as count';
// Handle different segments
if (event.segment === 'user') {
sb.select.count = 'countDistinct(profile_id) as count';
}
if (event.segment === 'session') {
sb.select.count = 'countDistinct(session_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
}
const mathFunction = {
property_sum: 'sum',
property_average: 'avg',
property_max: 'max',
property_min: 'min',
}[event.segment as string];
if (mathFunction && event.property) {
const propertyKey = getSelectPropertyKey(event.property);
if (isNumericColumn(event.property)) {
sb.select.count = `${mathFunction}(${propertyKey}) as count`;
sb.where.property = `${propertyKey} IS NOT NULL`;
} else {
sb.select.count = `${mathFunction}(toFloat64OrNull(${propertyKey})) as count`;
sb.where.property = `${propertyKey} IS NOT NULL AND notEmpty(${propertyKey})`;
}
}
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
sb.where,
' AND ',
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
sb.joins = {};
const sql = getSql();
console.log('-- Aggregate Chart --');
console.log(sql.replaceAll(/[\n\r]/g, ' '));
console.log('-- End --');
return sql;
}
// Order by count DESC (biggest first) for aggregate charts
sb.orderBy.count = 'count DESC';
// Apply limit if specified
if (limit) {
sb.limit = limit;
}
const sql = getSql();
console.log('-- Aggregate Chart --');
console.log(sql.replaceAll(/[\n\r]/g, ' '));
console.log('-- End --');
return sql;
}
function isNumericColumn(columnName: string): boolean {
const numericColumns = ['duration', 'revenue', 'longitude', 'latitude'];
return numericColumns.includes(columnName);

View File

@@ -11,6 +11,12 @@ import { getEventFiltersWhereClause } from './chart.service';
// Constants
const ROLLUP_DATE_PREFIX = '1970-01-01';
// Toggle revenue tracking in overview queries
const INCLUDE_REVENUE = true; // TODO: Make this configurable later
// Maximum number of records to return (for detail modals)
const MAX_RECORDS_LIMIT = 1000;
const COLUMN_PREFIX_MAP: Record<string, string> = {
region: 'country',
city: 'country',
@@ -47,8 +53,6 @@ export const zGetTopPagesInput = z.object({
filters: z.array(z.any()),
startDate: z.string(),
endDate: z.string(),
cursor: z.number().optional(),
limit: z.number().optional(),
});
export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput> & {
@@ -61,8 +65,6 @@ export const zGetTopEntryExitInput = z.object({
startDate: z.string(),
endDate: z.string(),
mode: z.enum(['entry', 'exit']),
cursor: z.number().optional(),
limit: z.number().optional(),
});
export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput> & {
@@ -97,14 +99,20 @@ export const zGetTopGenericInput = z.object({
'os',
'os_version',
]),
cursor: z.number().optional(),
limit: z.number().optional(),
});
export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput> & {
timezone: string;
};
export const zGetTopGenericSeriesInput = zGetTopGenericInput.extend({
interval: zTimeInterval,
});
export type IGetTopGenericSeriesInput = z.infer<typeof zGetTopGenericSeriesInput> & {
timezone: string;
};
export const zGetUserJourneyInput = z.object({
projectId: z.string(),
filters: z.array(z.any()),
@@ -543,18 +551,27 @@ export class OverviewService {
filters,
startDate,
endDate,
cursor = 1,
limit = 10,
timezone,
}: IGetTopPagesInput) {
const pageStatsQuery = clix(this.client, timezone)
.select([
'origin',
'path',
`last_value(properties['__title']) as title`,
'uniq(session_id) as count',
'round(avg(duration)/1000, 2) as avg_duration',
])
const selectColumns: (string | null | undefined | false)[] = [
'origin',
'path',
'uniq(session_id) as sessions',
'count() as pageviews',
];
if (INCLUDE_REVENUE) {
selectColumns.push('sum(revenue) as revenue');
}
const query = clix(this.client, timezone)
.select<{
origin: string;
path: string;
sessions: number;
pageviews: number;
revenue?: number;
}>(selectColumns)
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
@@ -563,57 +580,12 @@ export class OverviewService {
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters))
.groupBy(['origin', 'path'])
.orderBy('count', 'DESC')
.limit(limit)
.offset((cursor - 1) * limit);
const bounceStatsQuery = clix(this.client, timezone)
.select([
'entry_path',
'entry_origin',
'coalesce(round(countIf(is_bounce = 1 AND sign = 1) * 100.0 / countIf(sign = 1), 2), 0) as bounce_rate',
])
.from(TABLE_NAMES.sessions, true)
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy(['entry_path', 'entry_origin']);
pageStatsQuery.rawWhere(this.getRawWhereClause('events', filters));
bounceStatsQuery.rawWhere(this.getRawWhereClause('sessions', filters));
const mainQuery = clix(this.client, timezone)
.with('page_stats', pageStatsQuery)
.with('bounce_stats', bounceStatsQuery)
.select<{
title: string;
origin: string;
path: string;
avg_duration: number;
bounce_rate: number;
sessions: number;
revenue: number;
}>([
'p.title',
'p.origin',
'p.path',
'p.avg_duration',
'p.count as sessions',
'b.bounce_rate',
])
.from('page_stats p', false)
.leftJoin(
'bounce_stats b',
'p.path = b.entry_path AND p.origin = b.entry_origin',
)
.orderBy('sessions', 'DESC')
.limit(limit);
.limit(MAX_RECORDS_LIMIT);
return mainQuery.execute();
return query.execute();
}
async getTopEntryExit({
@@ -622,28 +594,27 @@ export class OverviewService {
startDate,
endDate,
mode,
cursor = 1,
limit = 10,
timezone,
}: IGetTopEntryExitInput) {
const offset = (cursor - 1) * limit;
const selectColumns: (string | null | undefined | false)[] = [
`${mode}_origin AS origin`,
`${mode}_path AS path`,
'sum(sign) as sessions',
'sum(sign * screen_view_count) as pageviews',
];
if (INCLUDE_REVENUE) {
selectColumns.push('sum(revenue * sign) as revenue');
}
const query = clix(this.client, timezone)
.select<{
origin: string;
path: string;
avg_duration: number;
bounce_rate: number;
sessions: number;
revenue: number;
}>([
`${mode}_origin AS origin`,
`${mode}_path AS path`,
'round(avg(duration * sign)/1000, 2) as avg_duration',
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
'sum(sign) as sessions',
'sum(revenue * sign) as revenue',
])
pageviews: number;
revenue?: number;
}>(selectColumns)
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
@@ -653,8 +624,7 @@ export class OverviewService {
.groupBy([`${mode}_origin`, `${mode}_path`])
.having('sum(sign)', '>', 0)
.orderBy('sessions', 'DESC')
.limit(limit)
.offset(offset);
.limit(MAX_RECORDS_LIMIT);
const mainQuery = this.withDistinctSessionsIfNeeded(query, {
projectId,
@@ -697,29 +667,29 @@ export class OverviewService {
startDate,
endDate,
column,
cursor = 1,
limit = 10,
timezone,
}: IGetTopGenericInput) {
const prefixColumn = COLUMN_PREFIX_MAP[column] ?? null;
const offset = (cursor - 1) * limit;
const selectColumns: (string | null | undefined | false)[] = [
prefixColumn && `${prefixColumn} as prefix`,
`nullIf(${column}, '') as name`,
'sum(sign) as sessions',
'sum(sign * screen_view_count) as pageviews',
];
if (INCLUDE_REVENUE) {
selectColumns.push('sum(revenue * sign) as revenue');
}
const query = clix(this.client, timezone)
.select<{
prefix?: string;
name: string;
sessions: number;
bounce_rate: number;
avg_session_duration: number;
revenue: number;
}>([
prefixColumn && `${prefixColumn} as prefix`,
`nullIf(${column}, '') as name`,
'sum(sign) as sessions',
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) AS bounce_rate',
'round(avgIf(duration, duration > 0 AND sign > 0), 2)/1000 AS avg_session_duration',
'sum(revenue * sign) as revenue',
])
pageviews: number;
revenue?: number;
}>(selectColumns)
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
@@ -729,8 +699,7 @@ export class OverviewService {
.groupBy([prefixColumn, column].filter(Boolean))
.having('sum(sign)', '>', 0)
.orderBy('sessions', 'DESC')
.limit(limit)
.offset(offset);
.limit(MAX_RECORDS_LIMIT);
const mainQuery = this.withDistinctSessionsIfNeeded(query, {
projectId,
@@ -743,6 +712,177 @@ export class OverviewService {
return mainQuery.execute();
}
async getTopGenericSeries({
projectId,
filters,
startDate,
endDate,
column,
interval,
timezone,
}: IGetTopGenericSeriesInput): Promise<{
items: Array<{
name: string;
prefix?: string;
data: Array<{
date: string;
sessions: number;
pageviews: number;
revenue?: number;
}>;
total: { sessions: number; pageviews: number; revenue?: number };
}>;
}> {
const prefixColumn = COLUMN_PREFIX_MAP[column] ?? null;
const TOP_LIMIT = 15;
const fillConfig = this.getFillConfig(interval, startDate, endDate);
// Step 1: Get top 15 items
const selectColumns: (string | null | undefined | false)[] = [
prefixColumn && `${prefixColumn} as prefix`,
`nullIf(${column}, '') as name`,
'sum(sign) as sessions',
'sum(sign * screen_view_count) as pageviews',
];
if (INCLUDE_REVENUE) {
selectColumns.push('sum(revenue * sign) as revenue');
}
const topItemsQuery = clix(this.client, timezone)
.select<{
prefix?: string;
name: string;
sessions: number;
pageviews: number;
revenue?: number;
}>(selectColumns)
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy([prefixColumn, column].filter(Boolean))
.having('sum(sign)', '>', 0)
.orderBy('sessions', 'DESC')
.limit(TOP_LIMIT);
const mainTopItemsQuery = this.withDistinctSessionsIfNeeded(topItemsQuery, {
projectId,
filters,
startDate,
endDate,
timezone,
});
const topItems = await mainTopItemsQuery.execute();
if (topItems.length === 0) {
return { items: [] };
}
// Step 2: Build time-series query for each top item
const where = this.getRawWhereClause('sessions', filters);
const timeSeriesSelectColumns: (string | null | undefined | false)[] = [
`${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
prefixColumn && `${prefixColumn} as prefix`,
`nullIf(${column}, '') as name`,
'sum(sign) as sessions',
'sum(sign * screen_view_count) as pageviews',
];
if (INCLUDE_REVENUE) {
timeSeriesSelectColumns.push('sum(revenue * sign) as revenue');
}
const timeSeriesQuery = clix(this.client, timezone)
.select<{
date: string;
prefix?: string;
name: string;
sessions: number;
pageviews: number;
revenue?: number;
}>(timeSeriesSelectColumns)
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(where)
.groupBy(['date', prefixColumn, column].filter(Boolean))
.having('sum(sign)', '>', 0)
.orderBy('date', 'ASC')
.fill(fillConfig.from, fillConfig.to, fillConfig.step)
.transform({
date: (item) => new Date(item.date).toISOString(),
});
const mainTimeSeriesQuery = this.withDistinctSessionsIfNeeded(
timeSeriesQuery,
{
projectId,
filters,
startDate,
endDate,
timezone,
},
);
const timeSeriesData = await mainTimeSeriesQuery.execute();
// Step 3: Group time-series data by item and calculate totals
const itemsMap = new Map<
string,
{
name: string;
prefix?: string;
data: Array<{
date: string;
sessions: number;
pageviews: number;
revenue?: number;
}>;
total: { sessions: number; pageviews: number; revenue?: number };
}
>();
// Initialize items from topItems
for (const item of topItems) {
const key = `${item.prefix || ''}:${item.name}`;
itemsMap.set(key, {
name: item.name,
prefix: item.prefix,
data: [],
total: {
sessions: item.sessions,
pageviews: item.pageviews,
revenue: item.revenue ?? 0,
},
});
}
// Populate time-series data
for (const row of timeSeriesData) {
const key = `${row.prefix || ''}:${row.name}`;
const item = itemsMap.get(key);
if (item) {
item.data.push({
date: row.date,
sessions: row.sessions,
pageviews: row.pageviews,
revenue: row.revenue,
});
}
}
return {
items: Array.from(itemsMap.values()),
};
}
async getUserJourney({
projectId,
filters,

View File

@@ -0,0 +1,96 @@
import { TABLE_NAMES, ch } from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
export interface IGetTopPagesInput {
projectId: string;
startDate: string;
endDate: string;
timezone: string;
search?: string;
}
export interface ITopPage {
origin: string;
path: string;
title: string;
sessions: number;
pageviews: number;
avg_duration: number;
bounce_rate: number;
}
export class PagesService {
constructor(private client: typeof ch) {}
async getTopPages({
projectId,
startDate,
endDate,
timezone,
search,
}: IGetTopPagesInput): Promise<ITopPage[]> {
// CTE: Get titles from the last 30 days for faster retrieval
const titlesCte = clix(this.client, timezone)
.select([
'concat(origin, path) as page_key',
"anyLast(properties['__title']) as title",
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('created_at', '>=', clix.exp('now() - INTERVAL 30 DAY'))
.groupBy(['origin', 'path']);
// Pre-filtered sessions subquery for better performance
const sessionsSubquery = clix(this.client, timezone)
.select(['id', 'project_id', 'is_bounce'])
.from(TABLE_NAMES.sessions, true) // FINAL
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.where('sign', '=', 1);
// Main query: aggregate events and calculate bounce rate from pre-filtered sessions
const query = clix(this.client, timezone)
.with('page_titles', titlesCte)
.select<ITopPage>([
'e.origin as origin',
'e.path as path',
"coalesce(pt.title, '') as title",
'uniq(e.session_id) as sessions',
'count() as pageviews',
'round(avg(e.duration) / 1000 / 60, 2) as avg_duration',
`round(
(uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /
nullIf(uniq(e.session_id), 0),
2
) as bounce_rate`,
])
.from(`${TABLE_NAMES.events} e`, false)
.leftJoin(
sessionsSubquery,
'e.session_id = s.id AND e.project_id = s.project_id',
's',
)
.leftJoin('page_titles pt', 'concat(e.origin, e.path) = pt.page_key')
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.path', '!=', '')
.where('e.created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.when(!!search, (q) => {
q.where('e.path', 'LIKE', `%${search}%`);
})
.groupBy(['e.origin', 'e.path', 'pt.title'])
.orderBy('sessions', 'DESC')
.limit(1000);
return query.execute();
}
}
export const pagesService = new PagesService(ch);