zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(api): handle profile filters/breakdowns better

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-06-11 22:10:43 +02:00
parent 82239a7d9a
commit e3e9e60b25
3 changed files with 88 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/db/src/clickhouse/query-builder.ts
View File

@@ -24,7 +24,7 @@ type CTE = {
query: Query | string;
};
type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS';
type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS' | 'LEFT ANY';
type WhereCondition = {
condition: string;
@@ -61,7 +61,7 @@ export class Query<T = any> {
private _ctes: CTE[] = [];
private _joins: {
type: JoinType;
table: string | Expression;
table: string | Expression | Query;
condition: string;
alias?: string;
}[] = [];
@@ -280,13 +280,21 @@ export class Query<T = any> {
}
leftJoin(
table: string | Expression,
table: string | Expression | Query,
condition: string,
alias?: string,
): this {
return this.joinWithType('LEFT', table, condition, alias);
}
leftAnyJoin(
table: string | Expression | Query,
condition: string,
alias?: string,
): this {
return this.joinWithType('LEFT ANY', table, condition, alias);
}
rightJoin(
table: string | Expression,
condition: string,
@@ -309,7 +317,7 @@ export class Query<T = any> {
private joinWithType(
type: JoinType,
table: string | Expression,
table: string | Expression | Query,
condition: string,
alias?: string,
): this {
@@ -382,7 +390,7 @@ export class Query<T = any> {
const aliasClause = join.alias ? ` ${join.alias} ` : ' ';
const conditionStr = join.condition ? `ON ${join.condition}` : '';
parts.push(
`${join.type} JOIN ${join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`,
`${join.type} JOIN ${join.table instanceof Query ? `(${join.table.toSQL()})` : join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`,
);
});
}

23
packages/db/src/services/chart.service.ts
View File

@@ -31,6 +31,10 @@ export function transformPropertyKey(property: string) {
}
export function getSelectPropertyKey(property: string) {
if (property === 'has_profile') {
return `if(profile_id != device_id, 'true', 'false')`;
}
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
@@ -87,7 +91,13 @@ export function getChartSql({
);
if (anyFilterOnProfile || anyBreakdownOnProfile) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT * FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`;
sb.joins.profiles = `LEFT ANY JOIN (SELECT
id as "profile.id",
email as "profile.email",
first_name as "profile.first_name",
last_name as "profile.last_name",
properties as "profile.properties"
FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`;
}
sb.select.count = 'count(*) as count';
@@ -182,20 +192,25 @@ export function getChartSql({
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} WHERE ${join(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
sb.where,
' AND ',
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
sb.joins = {};
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql);
console.log('-- Report --');
console.log(sql.replaceAll(/[\n\r]/g, ' '));
console.log('-- End --');
return sql;
}
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql);
console.log('-- Report --');
console.log(sql.replaceAll(/[\n\r]/g, ' '));
console.log('-- End --');
return sql;
}

90
packages/trpc/src/routers/chart.ts
View File

@@ -3,6 +3,7 @@ import { escape } from 'sqlstring';
import { z } from 'zod';
import {
type IClickhouseProfile,
type IServiceProfile,
TABLE_NAMES,
ch,
@@ -97,7 +98,7 @@ export const chartRouter = createTRPCRouter({
.where('project_id', '=', projectId)
.where('is_external', '=', true)
.orderBy('created_at', 'DESC')
.limit(100)
.limit(10000)
.execute();
const profileProperties: string[] = [];
@@ -109,16 +110,21 @@ export const chartRouter = createTRPCRouter({
}
}
const res = await chQuery<{ property_key: string; created_at: string }>(
`SELECT
distinct property_key,
max(created_at) as created_at
FROM ${TABLE_NAMES.event_property_values_mv}
WHERE project_id = ${escape(projectId)}
${event && event !== '*' ? `AND name = ${escape(event)}` : ''}
GROUP BY property_key
ORDER BY created_at DESC`,
);
const query = clix(ch)
.select<{ property_key: string; created_at: string }>([
'distinct property_key',
'max(created_at) as created_at',
])
.from(TABLE_NAMES.event_property_values_mv)
.where('project_id', '=', projectId)
.groupBy(['property_key'])
.orderBy('created_at', 'DESC');
if (event && event !== '*') {
query.where('name', '=', event);
}
const res = await query.execute();
const properties = res
.map((item) => item.property_key)
@@ -179,35 +185,51 @@ export const chartRouter = createTRPCRouter({
const values: string[] = [];
if (property.startsWith('properties.')) {
const propertyKey = property.replace(/^properties\./, '');
const query = clix(ch)
.select<{
property_value: string;
created_at: string;
}>(['distinct property_value', 'max(created_at) as created_at'])
.from(TABLE_NAMES.event_property_values_mv)
.where('project_id', '=', projectId)
.where('property_key', '=', property.replace(/^properties\./, ''))
.groupBy(['property_value'])
.orderBy('created_at', 'DESC');
const res = await chQuery<{
property_value: string;
created_at: string;
}>(
`SELECT
distinct property_value,
max(created_at) as created_at
FROM ${TABLE_NAMES.event_property_values_mv}
WHERE project_id = ${escape(projectId)}
AND property_key = ${escape(propertyKey)}
${event && event !== '*' ? `AND name = ${escape(event)}` : ''}
GROUP BY property_value
ORDER BY created_at DESC`,
);
if (event && event !== '*') {
query.where('name', '=', event);
}
const res = await query.execute();
values.push(...res.map((e) => e.property_value));
} else {
const { sb, getSql } = createSqlBuilder();
sb.where.project_id = `project_id = ${escape(projectId)}`;
const query = clix(ch)
.select<{ values: string[] }>([
`distinct ${getSelectPropertyKey(property)} as values`,
])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('created_at', '>', clix.exp('now() - INTERVAL 6 MONTH'))
.orderBy('created_at', 'DESC')
.limit(100_000);
if (event !== '*') {
sb.where.event = `name = ${escape(event)}`;
query.where('name', '=', event);
}
sb.select.values = `distinct ${getSelectPropertyKey(property)} as values`;
sb.where.date = `${toDate('created_at', 'month')} > now() - INTERVAL 6 MONTH`;
sb.orderBy.created_at = 'created_at DESC';
sb.limit = 100_000;
const events = await chQuery<{ values: string[] }>(getSql());
if (property.startsWith('profile.')) {
query.leftAnyJoin(
clix(ch)
.select<IClickhouseProfile>([])
.from(TABLE_NAMES.profiles)
.where('project_id', '=', projectId),
'profile.id = profile_id',
'profile',
);
}
const events = await query.execute();
values.push(
...pipe(