diff --git a/packages/db/src/clickhouse/query-builder.ts b/packages/db/src/clickhouse/query-builder.ts index 3d8eaa51..67da224f 100644 --- a/packages/db/src/clickhouse/query-builder.ts +++ b/packages/db/src/clickhouse/query-builder.ts @@ -24,7 +24,7 @@ type CTE = { query: Query | string; }; -type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS'; +type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS' | 'LEFT ANY'; type WhereCondition = { condition: string; @@ -61,7 +61,7 @@ export class Query { private _ctes: CTE[] = []; private _joins: { type: JoinType; - table: string | Expression; + table: string | Expression | Query; condition: string; alias?: string; }[] = []; @@ -280,13 +280,21 @@ export class Query { } leftJoin( - table: string | Expression, + table: string | Expression | Query, condition: string, alias?: string, ): this { return this.joinWithType('LEFT', table, condition, alias); } + leftAnyJoin( + table: string | Expression | Query, + condition: string, + alias?: string, + ): this { + return this.joinWithType('LEFT ANY', table, condition, alias); + } + rightJoin( table: string | Expression, condition: string, @@ -309,7 +317,7 @@ export class Query { private joinWithType( type: JoinType, - table: string | Expression, + table: string | Expression | Query, condition: string, alias?: string, ): this { @@ -382,7 +390,7 @@ export class Query { const aliasClause = join.alias ? ` ${join.alias} ` : ' '; const conditionStr = join.condition ? `ON ${join.condition}` : ''; parts.push( - `${join.type} JOIN ${join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`, + `${join.type} JOIN ${join.table instanceof Query ? `(${join.table.toSQL()})` : join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`, ); }); } diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts index 68ee9512..fc53a402 100644 --- a/packages/db/src/services/chart.service.ts +++ b/packages/db/src/services/chart.service.ts @@ -31,6 +31,10 @@ export function transformPropertyKey(property: string) { } export function getSelectPropertyKey(property: string) { + if (property === 'has_profile') { + return `if(profile_id != device_id, 'true', 'false')`; + } + const propertyPatterns = ['properties', 'profile.properties']; const match = propertyPatterns.find((pattern) => @@ -87,7 +91,13 @@ export function getChartSql({ ); if (anyFilterOnProfile || anyBreakdownOnProfile) { - sb.joins.profiles = `LEFT ANY JOIN (SELECT * FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`; + sb.joins.profiles = `LEFT ANY JOIN (SELECT + id as "profile.id", + email as "profile.email", + first_name as "profile.first_name", + last_name as "profile.last_name", + properties as "profile.properties" + FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`; } sb.select.count = 'count(*) as count'; @@ -182,20 +192,25 @@ export function getChartSql({ if (event.segment === 'one_event_per_user') { sb.from = `( - SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} WHERE ${join( + SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join( sb.where, ' AND ', )} ORDER BY profile_id, created_at DESC ) as subQuery`; + sb.joins = {}; const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`; - console.log('CHART SQL', sql); + console.log('-- Report --'); + console.log(sql.replaceAll(/[\n\r]/g, ' ')); + console.log('-- End --'); return sql; } const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`; - console.log('CHART SQL', sql); + console.log('-- Report --'); + console.log(sql.replaceAll(/[\n\r]/g, ' ')); + console.log('-- End --'); return sql; } diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 775dd754..922d4b37 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -3,6 +3,7 @@ import { escape } from 'sqlstring'; import { z } from 'zod'; import { + type IClickhouseProfile, type IServiceProfile, TABLE_NAMES, ch, @@ -97,7 +98,7 @@ export const chartRouter = createTRPCRouter({ .where('project_id', '=', projectId) .where('is_external', '=', true) .orderBy('created_at', 'DESC') - .limit(100) + .limit(10000) .execute(); const profileProperties: string[] = []; @@ -109,16 +110,21 @@ export const chartRouter = createTRPCRouter({ } } - const res = await chQuery<{ property_key: string; created_at: string }>( - `SELECT - distinct property_key, - max(created_at) as created_at - FROM ${TABLE_NAMES.event_property_values_mv} - WHERE project_id = ${escape(projectId)} - ${event && event !== '*' ? `AND name = ${escape(event)}` : ''} - GROUP BY property_key - ORDER BY created_at DESC`, - ); + const query = clix(ch) + .select<{ property_key: string; created_at: string }>([ + 'distinct property_key', + 'max(created_at) as created_at', + ]) + .from(TABLE_NAMES.event_property_values_mv) + .where('project_id', '=', projectId) + .groupBy(['property_key']) + .orderBy('created_at', 'DESC'); + + if (event && event !== '*') { + query.where('name', '=', event); + } + + const res = await query.execute(); const properties = res .map((item) => item.property_key) @@ -179,35 +185,51 @@ export const chartRouter = createTRPCRouter({ const values: string[] = []; if (property.startsWith('properties.')) { - const propertyKey = property.replace(/^properties\./, ''); + const query = clix(ch) + .select<{ + property_value: string; + created_at: string; + }>(['distinct property_value', 'max(created_at) as created_at']) + .from(TABLE_NAMES.event_property_values_mv) + .where('project_id', '=', projectId) + .where('property_key', '=', property.replace(/^properties\./, '')) + .groupBy(['property_value']) + .orderBy('created_at', 'DESC'); - const res = await chQuery<{ - property_value: string; - created_at: string; - }>( - `SELECT - distinct property_value, - max(created_at) as created_at - FROM ${TABLE_NAMES.event_property_values_mv} - WHERE project_id = ${escape(projectId)} - AND property_key = ${escape(propertyKey)} - ${event && event !== '*' ? `AND name = ${escape(event)}` : ''} - GROUP BY property_value - ORDER BY created_at DESC`, - ); + if (event && event !== '*') { + query.where('name', '=', event); + } + + const res = await query.execute(); values.push(...res.map((e) => e.property_value)); } else { - const { sb, getSql } = createSqlBuilder(); - sb.where.project_id = `project_id = ${escape(projectId)}`; + const query = clix(ch) + .select<{ values: string[] }>([ + `distinct ${getSelectPropertyKey(property)} as values`, + ]) + .from(TABLE_NAMES.events) + .where('project_id', '=', projectId) + .where('created_at', '>', clix.exp('now() - INTERVAL 6 MONTH')) + .orderBy('created_at', 'DESC') + .limit(100_000); + if (event !== '*') { - sb.where.event = `name = ${escape(event)}`; + query.where('name', '=', event); } - sb.select.values = `distinct ${getSelectPropertyKey(property)} as values`; - sb.where.date = `${toDate('created_at', 'month')} > now() - INTERVAL 6 MONTH`; - sb.orderBy.created_at = 'created_at DESC'; - sb.limit = 100_000; - const events = await chQuery<{ values: string[] }>(getSql()); + + if (property.startsWith('profile.')) { + query.leftAnyJoin( + clix(ch) + .select([]) + .from(TABLE_NAMES.profiles) + .where('project_id', '=', projectId), + 'profile.id = profile_id', + 'profile', + ); + } + + const events = await query.execute(); values.push( ...pipe(