zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(api): handle profile filters/breakdowns better

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-06-11 22:10:43 +02:00
parent 82239a7d9a
commit e3e9e60b25
3 changed files with 88 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/db/src/clickhouse/query-builder.ts
View File

@@ -24,7 +24,7 @@ type CTE = {
query: Query | string; query: Query | string;
}; };
type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS'; type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS' | 'LEFT ANY';
type WhereCondition = { type WhereCondition = {
condition: string; condition: string;
@@ -61,7 +61,7 @@ export class Query<T = any> {
private _ctes: CTE[] = []; private _ctes: CTE[] = [];
private _joins: { private _joins: {
type: JoinType; type: JoinType;
table: string | Expression; table: string | Expression | Query;
condition: string; condition: string;
alias?: string; alias?: string;
}[] = []; }[] = [];
@@ -280,13 +280,21 @@ export class Query<T = any> {
} }
leftJoin( leftJoin(
table: string | Expression, table: string | Expression | Query,
condition: string, condition: string,
alias?: string, alias?: string,
): this { ): this {
return this.joinWithType('LEFT', table, condition, alias); return this.joinWithType('LEFT', table, condition, alias);
} }
leftAnyJoin(
table: string | Expression | Query,
condition: string,
alias?: string,
): this {
return this.joinWithType('LEFT ANY', table, condition, alias);
}
rightJoin( rightJoin(
table: string | Expression, table: string | Expression,
condition: string, condition: string,
@@ -309,7 +317,7 @@ export class Query<T = any> {
private joinWithType( private joinWithType(
type: JoinType, type: JoinType,
table: string | Expression, table: string | Expression | Query,
condition: string, condition: string,
alias?: string, alias?: string,
): this { ): this {
@@ -382,7 +390,7 @@ export class Query<T = any> {
const aliasClause = join.alias ? ` ${join.alias} ` : ' '; const aliasClause = join.alias ? ` ${join.alias} ` : ' ';
const conditionStr = join.condition ? `ON ${join.condition}` : ''; const conditionStr = join.condition ? `ON ${join.condition}` : '';
parts.push( parts.push(
`${join.type} JOIN ${join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`, `${join.type} JOIN ${join.table instanceof Query ? `(${join.table.toSQL()})` : join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`,
); );
}); });
} }

23
packages/db/src/services/chart.service.ts
View File

@@ -31,6 +31,10 @@ export function transformPropertyKey(property: string) {
} }
export function getSelectPropertyKey(property: string) { export function getSelectPropertyKey(property: string) {
if (property === 'has_profile') {
return `if(profile_id != device_id, 'true', 'false')`;
}
const propertyPatterns = ['properties', 'profile.properties']; const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) => const match = propertyPatterns.find((pattern) =>
@@ -87,7 +91,13 @@ export function getChartSql({
); );
if (anyFilterOnProfile || anyBreakdownOnProfile) { if (anyFilterOnProfile || anyBreakdownOnProfile) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT * FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`; sb.joins.profiles = `LEFT ANY JOIN (SELECT
id as "profile.id",
email as "profile.email",
first_name as "profile.first_name",
last_name as "profile.last_name",
properties as "profile.properties"
FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`;
} }
sb.select.count = 'count(*) as count'; sb.select.count = 'count(*) as count';
@@ -182,20 +192,25 @@ export function getChartSql({
if (event.segment === 'one_event_per_user') { if (event.segment === 'one_event_per_user') {
sb.from = `( sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} WHERE ${join( SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
sb.where, sb.where,
' AND ', ' AND ',
)} )}
ORDER BY profile_id, created_at DESC ORDER BY profile_id, created_at DESC
) as subQuery`; ) as subQuery`;
sb.joins = {};
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`; const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql); console.log('-- Report --');
console.log(sql.replaceAll(/[\n\r]/g, ' '));
console.log('-- End --');
return sql; return sql;
} }
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`; const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql); console.log('-- Report --');
console.log(sql.replaceAll(/[\n\r]/g, ' '));
console.log('-- End --');
return sql; return sql;
} }

88
packages/trpc/src/routers/chart.ts
View File

@@ -3,6 +3,7 @@ import { escape } from 'sqlstring';
import { z } from 'zod'; import { z } from 'zod';
import { import {
type IClickhouseProfile,
type IServiceProfile, type IServiceProfile,
TABLE_NAMES, TABLE_NAMES,
ch, ch,
@@ -97,7 +98,7 @@ export const chartRouter = createTRPCRouter({
.where('project_id', '=', projectId) .where('project_id', '=', projectId)
.where('is_external', '=', true) .where('is_external', '=', true)
.orderBy('created_at', 'DESC') .orderBy('created_at', 'DESC')
.limit(100) .limit(10000)
.execute(); .execute();
const profileProperties: string[] = []; const profileProperties: string[] = [];
@@ -109,16 +110,21 @@ export const chartRouter = createTRPCRouter({
} }
} }
const res = await chQuery<{ property_key: string; created_at: string }>( const query = clix(ch)
`SELECT .select<{ property_key: string; created_at: string }>([
distinct property_key, 'distinct property_key',
max(created_at) as created_at 'max(created_at) as created_at',
FROM ${TABLE_NAMES.event_property_values_mv} ])
WHERE project_id = ${escape(projectId)} .from(TABLE_NAMES.event_property_values_mv)
${event && event !== '*' ? `AND name = ${escape(event)}` : ''} .where('project_id', '=', projectId)
GROUP BY property_key .groupBy(['property_key'])
ORDER BY created_at DESC`, .orderBy('created_at', 'DESC');
);
if (event && event !== '*') {
query.where('name', '=', event);
}
const res = await query.execute();
const properties = res const properties = res
.map((item) => item.property_key) .map((item) => item.property_key)
@@ -179,35 +185,51 @@ export const chartRouter = createTRPCRouter({
const values: string[] = []; const values: string[] = [];
if (property.startsWith('properties.')) { if (property.startsWith('properties.')) {
const propertyKey = property.replace(/^properties\./, ''); const query = clix(ch)
.select<{
const res = await chQuery<{
property_value: string; property_value: string;
created_at: string; created_at: string;
}>( }>(['distinct property_value', 'max(created_at) as created_at'])
`SELECT .from(TABLE_NAMES.event_property_values_mv)
distinct property_value, .where('project_id', '=', projectId)
max(created_at) as created_at .where('property_key', '=', property.replace(/^properties\./, ''))
FROM ${TABLE_NAMES.event_property_values_mv} .groupBy(['property_value'])
WHERE project_id = ${escape(projectId)} .orderBy('created_at', 'DESC');
AND property_key = ${escape(propertyKey)}
${event && event !== '*' ? `AND name = ${escape(event)}` : ''} if (event && event !== '*') {
GROUP BY property_value query.where('name', '=', event);
ORDER BY created_at DESC`, }
);
const res = await query.execute();
values.push(...res.map((e) => e.property_value)); values.push(...res.map((e) => e.property_value));
} else { } else {
const { sb, getSql } = createSqlBuilder(); const query = clix(ch)
sb.where.project_id = `project_id = ${escape(projectId)}`; .select<{ values: string[] }>([
`distinct ${getSelectPropertyKey(property)} as values`,
])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('created_at', '>', clix.exp('now() - INTERVAL 6 MONTH'))
.orderBy('created_at', 'DESC')
.limit(100_000);
if (event !== '*') { if (event !== '*') {
sb.where.event = `name = ${escape(event)}`; query.where('name', '=', event);
} }
sb.select.values = `distinct ${getSelectPropertyKey(property)} as values`;
sb.where.date = `${toDate('created_at', 'month')} > now() - INTERVAL 6 MONTH`; if (property.startsWith('profile.')) {
sb.orderBy.created_at = 'created_at DESC'; query.leftAnyJoin(
sb.limit = 100_000; clix(ch)
const events = await chQuery<{ values: string[] }>(getSql()); .select<IClickhouseProfile>([])
.from(TABLE_NAMES.profiles)
.where('project_id', '=', projectId),
'profile.id = profile_id',
'profile',
);
}
const events = await query.execute();
values.push( values.push(
...pipe( ...pipe(