zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: group analytics

Browse Source 
* wip

* wip

* wip

* wip

* wip

* add buffer

* wip

* wip

* fixes

* fix

* wip

* group validation

* fix group issues

* docs: add groups
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-20 10:46:09 +01:00
committed by GitHub
parent 88a2d876ce
commit 11e9ecac1a
99 changed files with 5944 additions and 1432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

499
packages/db/src/services/chart.service.ts
View File

@@ -1,20 +1,19 @@
import sqlstring from 'sqlstring';
/** biome-ignore-all lint/style/useDefaultSwitchClause: switch cases are exhaustive by design */
import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common';
import type {
IChartEventFilter,
IReportInput,
IChartRange,
IGetChartDataInput,
IReportInput,
} from '@openpanel/validation';
import { TABLE_NAMES, formatClickhouseDate } from '../clickhouse/client';
import sqlstring from 'sqlstring';
import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
export function transformPropertyKey(property: string) {
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
property.startsWith(`${pattern}.`)
);
if (!match) {
@@ -32,21 +31,91 @@ export function transformPropertyKey(property: string) {
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
export function getSelectPropertyKey(property: string) {
// Returns a SQL expression for a group property via the _g JOIN alias
// property format: "group.name", "group.type", "group.properties.plan"
export function getGroupPropertySql(property: string): string {
const withoutPrefix = property.replace(/^group\./, '');
if (withoutPrefix === 'name') {
return '_g.name';
}
if (withoutPrefix === 'type') {
return '_g.type';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `_g.properties[${sqlstring.escape(propKey)}]`;
}
return '_group_id';
}
// Returns the SELECT expression when querying the groups table directly (no join alias).
// Use for fetching distinct values for group.* properties.
export function getGroupPropertySelect(property: string): string {
const withoutPrefix = property.replace(/^group\./, '');
if (withoutPrefix === 'name') {
return 'name';
}
if (withoutPrefix === 'type') {
return 'type';
}
if (withoutPrefix === 'id') {
return 'id';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `properties[${sqlstring.escape(propKey)}]`;
}
return 'id';
}
// Returns the SELECT expression when querying the profiles table directly (no join alias).
// Use for fetching distinct values for profile.* properties.
export function getProfilePropertySelect(property: string): string {
const withoutPrefix = property.replace(/^profile\./, '');
if (withoutPrefix === 'id') {
return 'id';
}
if (withoutPrefix === 'first_name') {
return 'first_name';
}
if (withoutPrefix === 'last_name') {
return 'last_name';
}
if (withoutPrefix === 'email') {
return 'email';
}
if (withoutPrefix === 'avatar') {
return 'avatar';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `properties[${sqlstring.escape(propKey)}]`;
}
return 'id';
}
export function getSelectPropertyKey(property: string, projectId?: string) {
if (property === 'has_profile') {
return `if(profile_id != device_id, 'true', 'false')`;
}
// Handle group properties — requires ARRAY JOIN + _g JOIN to be present in query
if (property.startsWith('group.') && projectId) {
return getGroupPropertySql(property);
}
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
property.startsWith(`${pattern}.`)
);
if (!match) return property;
if (!match) {
return property;
}
if (property.includes('*')) {
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${sqlstring.escape(
transformPropertyKey(property),
transformPropertyKey(property)
)})))`;
}
@@ -60,9 +129,7 @@ export function getChartSql({
startDate,
endDate,
projectId,
limit,
timezone,
chartType,
}: IGetChartDataInput & { timezone: string }) {
const {
sb,
@@ -78,22 +145,43 @@ export function getChartSql({
with: addCte,
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
sb.where.eventName = `e.name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
filter.name.startsWith('profile.'),
filter.name.startsWith('profile.')
);
const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
breakdown.name.startsWith('profile.'),
breakdown.name.startsWith('profile.')
);
const anyFilterOnGroup = event.filters.some((filter) =>
filter.name.startsWith('group.')
);
const anyBreakdownOnGroup = breakdowns.some((breakdown) =>
breakdown.name.startsWith('group.')
);
const anyMetricOnGroup = !!event.property?.startsWith('group.');
const needsGroupArrayJoin =
anyFilterOnGroup ||
anyBreakdownOnGroup ||
anyMetricOnGroup ||
event.segment === 'group';
if (needsGroupArrayJoin) {
addCte(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_table = 'LEFT ANY JOIN _g ON _g.id = _group_id';
}
// Build WHERE clause without the bar filter (for use in subqueries and CTEs)
// Define this early so we can use it in CTE definitions
@@ -178,8 +266,8 @@ export function getChartSql({
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`
);
// Use the CTE reference in the main query
@@ -225,31 +313,11 @@ export function getChartSql({
sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
}
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
.map((b) => getSelectPropertyKey(b.name))
.join(', ');
// Add top_breakdowns CTE using the builder
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.select[key] =
`${getSelectPropertyKey(breakdown.name, projectId)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -261,6 +329,10 @@ export function getChartSql({
sb.select.count = 'countDistinct(session_id) as count';
}
if (event.segment === 'group') {
sb.select.count = 'countDistinct(_group_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
@@ -287,9 +359,9 @@ export function getChartSql({
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} e ${getJoins()} WHERE ${join(
sb.where,
' AND ',
' AND '
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
@@ -303,41 +375,52 @@ export function getChartSql({
}
// Note: The profile CTE (if it exists) is available in subqueries, so we can reference it directly
const subqueryGroupJoins = needsGroupArrayJoin
? 'ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id '
: '';
if (breakdowns.length > 0) {
// Match breakdown properties in subquery with outer query's grouped values
// Since outer query groups by label_X, we reference those in the correlation
const breakdownMatches = breakdowns
// Pre-compute unique counts per breakdown group in a CTE, then JOIN it.
// We can't use a correlated subquery because:
// 1. ClickHouse expands label_X aliases to their underlying expressions,
// which resolve in the subquery's scope, making the condition a tautology.
// 2. Correlated subqueries aren't supported on distributed/remote tables.
const ucSelectParts: string[] = breakdowns.map((breakdown, index) => {
const propertyKey = getSelectPropertyKey(breakdown.name, projectId);
return `${propertyKey} as _uc_label_${index + 1}`;
});
ucSelectParts.push('uniq(profile_id) as total_count');
const ucGroupByParts = breakdowns.map(
(_, index) => `_uc_label_${index + 1}`
);
const ucWhere = getWhereWithoutBar();
addCte(
'_uc',
`SELECT ${ucSelectParts.join(', ')} FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${ucWhere} GROUP BY ${ucGroupByParts.join(', ')}`
);
const ucJoinConditions = breakdowns
.map((b, index) => {
const propertyKey = getSelectPropertyKey(b.name);
// Correlate: match the property expression with outer query's label_X value
// ClickHouse allows referencing outer query columns in correlated subqueries
return `${propertyKey} = label_${index + 1}`;
const propertyKey = getSelectPropertyKey(b.name, projectId);
return `_uc._uc_label_${index + 1} = ${propertyKey}`;
})
.join(' AND ');
// Build WHERE clause for subquery - replace table alias and keep profile CTE reference
const subqueryWhere = getWhereWithoutBar()
.replace(/\be\./g, 'e2.')
.replace(/\bprofile\./g, 'profile.');
sb.select.total_unique_count = `(
SELECT uniq(profile_id)
FROM ${TABLE_NAMES.events} e2
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
AND ${breakdownMatches}
) as total_count`;
sb.joins.unique_counts = `LEFT ANY JOIN _uc ON ${ucJoinConditions}`;
sb.select.total_unique_count = 'any(_uc.total_count) as total_count';
} else {
// No breakdowns: calculate unique count across all data
// Build WHERE clause for subquery - replace table alias and keep profile CTE reference
const subqueryWhere = getWhereWithoutBar()
.replace(/\be\./g, 'e2.')
.replace(/\bprofile\./g, 'profile.');
const ucWhere = getWhereWithoutBar();
sb.select.total_unique_count = `(
SELECT uniq(profile_id)
FROM ${TABLE_NAMES.events} e2
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
) as total_count`;
addCte(
'_uc',
`SELECT uniq(profile_id) as total_count FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${ucWhere}`
);
sb.select.total_unique_count =
'(SELECT total_count FROM _uc) as total_count';
}
const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
@@ -359,31 +442,43 @@ export function getAggregateChartSql({
}) {
const { sb, join, getJoins, with: addCte, getSql } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
sb.where.eventName = `e.name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
filter.name.startsWith('profile.'),
filter.name.startsWith('profile.')
);
const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
breakdown.name.startsWith('profile.'),
breakdown.name.startsWith('profile.')
);
const anyFilterOnGroup = event.filters.some((filter) =>
filter.name.startsWith('group.')
);
const anyBreakdownOnGroup = breakdowns.some((breakdown) =>
breakdown.name.startsWith('group.')
);
const anyMetricOnGroup = !!event.property?.startsWith('group.');
const needsGroupArrayJoin =
anyFilterOnGroup ||
anyBreakdownOnGroup ||
anyMetricOnGroup ||
event.segment === 'group';
// Build WHERE clause without the bar filter (for use in subqueries and CTEs)
const getWhereWithoutBar = () => {
const whereWithoutBar = { ...sb.where };
delete whereWithoutBar.bar;
return Object.keys(whereWithoutBar).length
? `WHERE ${join(whereWithoutBar, ' AND ')}`
: '';
};
if (needsGroupArrayJoin) {
addCte(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_table = 'LEFT ANY JOIN _g ON _g.id = _group_id';
}
// Collect all profile fields used in filters and breakdowns
const getProfileFields = () => {
@@ -455,8 +550,8 @@ export function getAggregateChartSql({
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.profiles = profilesJoinRef;
@@ -475,31 +570,12 @@ export function getAggregateChartSql({
// Use startDate as the date value since we're aggregating across the entire range
sb.select.date = `${sqlstring.escape(startDate)} as date`;
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
.map((b) => getSelectPropertyKey(b.name))
.join(', ');
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
// Add breakdowns to SELECT and GROUP BY
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.select[key] =
`${getSelectPropertyKey(breakdown.name, projectId)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -518,6 +594,10 @@ export function getAggregateChartSql({
sb.select.count = 'countDistinct(session_id) as count';
}
if (event.segment === 'group') {
sb.select.count = 'countDistinct(_group_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
@@ -531,7 +611,7 @@ export function getAggregateChartSql({
}[event.segment as string];
if (mathFunction && event.property) {
const propertyKey = getSelectPropertyKey(event.property);
const propertyKey = getSelectPropertyKey(event.property, projectId);
if (isNumericColumn(event.property)) {
sb.select.count = `${mathFunction}(${propertyKey}) as count`;
@@ -544,9 +624,9 @@ export function getAggregateChartSql({
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} e ${getJoins()} WHERE ${join(
sb.where,
' AND ',
' AND '
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
@@ -579,7 +659,10 @@ function isNumericColumn(columnName: string): boolean {
return numericColumns.includes(columnName);
}
export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
export function getEventFiltersWhereClause(
filters: IChartEventFilter[],
projectId?: string
) {
const where: Record<string, string> = {};
filters.forEach((filter, index) => {
const id = `f${index}`;
@@ -602,6 +685,67 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
return;
}
// Handle group. prefixed filters (requires ARRAY JOIN + _g JOIN in query)
if (name.startsWith('group.') && projectId) {
const whereFrom = getGroupPropertySql(name);
switch (operator) {
case 'is': {
if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] =
`${whereFrom} IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')})`;
}
break;
}
case 'isNot': {
if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] =
`${whereFrom} NOT IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')})`;
}
break;
}
case 'contains': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'doesNotContain': {
where[id] =
`(${value.map((val) => `${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'startsWith': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'endsWith': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`).join(' OR ')})`;
break;
}
case 'isNull': {
where[id] = `(${whereFrom} = '' OR ${whereFrom} IS NULL)`;
break;
}
case 'isNotNull': {
where[id] = `(${whereFrom} != '' AND ${whereFrom} IS NOT NULL)`;
break;
}
case 'regex': {
where[id] =
`(${value.map((val) => `match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`).join(' OR ')})`;
break;
}
}
return;
}
if (
name.startsWith('properties.') ||
name.startsWith('profile.properties.')
@@ -616,15 +760,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x = ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
where[id] = `${whereFrom} IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
}
@@ -633,15 +775,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x != ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} NOT IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
where[id] = `${whereFrom} NOT IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
}
@@ -649,15 +789,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -668,14 +807,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -685,14 +824,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
(val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -702,14 +841,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')})`;
}
@@ -724,7 +863,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`,
`match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -752,14 +891,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -770,14 +909,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -788,14 +927,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -806,14 +945,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -856,7 +995,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -865,7 +1004,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -874,7 +1013,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
`${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -883,7 +1022,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')})`;
break;
@@ -892,7 +1031,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
`match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`
)
.join(' OR ')})`;
break;
@@ -902,7 +1041,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
@@ -917,7 +1056,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
@@ -932,13 +1071,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
where[id] = `(${value
.map(
(val) => `${name} >= ${sqlstring.escape(String(val).trim())}`,
(val) => `${name} >= ${sqlstring.escape(String(val).trim())}`
)
.join(' OR ')})`;
}
@@ -949,13 +1088,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
where[id] = `(${value
.map(
(val) => `${name} <= ${sqlstring.escape(String(val).trim())}`,
(val) => `${name} <= ${sqlstring.escape(String(val).trim())}`
)
.join(' OR ')})`;
}
@@ -974,15 +1113,15 @@ export function getChartStartEndDate(
endDate,
range,
}: Pick<IReportInput, 'endDate' | 'startDate' | 'range'>,
timezone: string,
timezone: string
) {
if (startDate && endDate) {
return { startDate: startDate, endDate: endDate };
return { startDate, endDate };
}
const ranges = getDatesFromRange(range, timezone);
if (!startDate && endDate) {
return { startDate: ranges.startDate, endDate: endDate };
return { startDate: ranges.startDate, endDate };
}
return ranges;
@@ -1002,8 +1141,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1018,8 +1157,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1035,8 +1174,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.endOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1053,8 +1192,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1071,8 +1210,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1089,8 +1228,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1106,8 +1245,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1124,8 +1263,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1141,8 +1280,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1152,8 +1291,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1170,8 +1309,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1183,7 +1322,7 @@ export function getChartPrevStartEndDate({
endDate: string;
}) {
let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff(
DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'),
DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss')
);
// this will make sure our start and end date's are correct

32
packages/db/src/services/conversion.service.ts
View File

@@ -31,14 +31,14 @@ export class ConversionService {
const funnelWindow = funnelOptions?.funnelWindow ?? 24;
const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';
const breakdownExpressions = breakdowns.map(
(b) => getSelectPropertyKey(b.name),
(b) => getSelectPropertyKey(b.name, projectId),
);
const breakdownSelects = breakdownExpressions.map(
(expr, index) => `${expr} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);
// Check if any breakdown uses profile fields and build profile JOIN if needed
// Check if any breakdown or filter uses profile fields
const profileBreakdowns = breakdowns.filter((b) =>
b.name.startsWith('profile.'),
);
@@ -71,6 +71,15 @@ export class ConversionService {
const events = onlyReportEvents(series);
// Check if any breakdown or filter uses group fields
const anyBreakdownOnGroup = breakdowns.some((b) =>
b.name.startsWith('group.'),
);
const anyFilterOnGroup = events.some((e) =>
e.filters?.some((f) => f.name.startsWith('group.')),
);
const needsGroupArrayJoin = anyBreakdownOnGroup || anyFilterOnGroup;
if (events.length !== 2) {
throw new Error('events must be an array of two events');
}
@@ -82,21 +91,25 @@ export class ConversionService {
const eventA = events[0]!;
const eventB = events[1]!;
const whereA = Object.values(
getEventFiltersWhereClause(eventA.filters),
getEventFiltersWhereClause(eventA.filters, projectId),
).join(' AND ');
const whereB = Object.values(
getEventFiltersWhereClause(eventB.filters),
getEventFiltersWhereClause(eventB.filters, projectId),
).join(' AND ');
const funnelWindowSeconds = funnelWindow * 3600;
// Build funnel conditions
const conditionA = whereA
? `(name = '${eventA.name}' AND ${whereA})`
: `name = '${eventA.name}'`;
? `(events.name = '${eventA.name}' AND ${whereA})`
: `events.name = '${eventA.name}'`;
const conditionB = whereB
? `(name = '${eventB.name}' AND ${whereB})`
: `name = '${eventB.name}'`;
? `(events.name = '${eventB.name}' AND ${whereB})`
: `events.name = '${eventB.name}'`;
const groupJoin = needsGroupArrayJoin
? `ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`
: '';
// Use windowFunnel approach - single scan, no JOIN
const query = clix(this.client, timezone)
@@ -126,8 +139,9 @@ export class ConversionService {
) as steps
FROM ${TABLE_NAMES.events}
${profileJoin}
${groupJoin}
WHERE project_id = '${projectId}'
AND name IN ('${eventA.name}', '${eventB.name}')
AND events.name IN ('${eventA.name}', '${eventB.name}')
AND created_at BETWEEN toDateTime('${startDate}') AND toDateTime('${endDate}')
GROUP BY ${group}${breakdownExpressions.length ? `, ${breakdownExpressions.join(', ')}` : ''})
`),

56
packages/db/src/services/event.service.ts
View File

@@ -32,14 +32,14 @@ export type IImportedEvent = Omit<
properties: Record<string, unknown>;
};
export type IServicePage = {
export interface IServicePage {
path: string;
count: number;
project_id: string;
first_seen: string;
title: string;
origin: string;
};
}
export interface IClickhouseBotEvent {
id: string;
@@ -92,6 +92,7 @@ export interface IClickhouseEvent {
sdk_name: string;
sdk_version: string;
revenue?: number;
groups: string[];
// They do not exist here. Just make ts happy for now
profile?: IServiceProfile;
@@ -143,6 +144,7 @@ export function transformSessionToEvent(
importedAt: undefined,
sdkName: undefined,
sdkVersion: undefined,
groups: [],
};
}
@@ -179,6 +181,7 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
sdkVersion: event.sdk_version,
profile: event.profile,
revenue: event.revenue,
groups: event.groups ?? [],
};
}
@@ -227,6 +230,7 @@ export interface IServiceEvent {
sdkName: string | undefined;
sdkVersion: string | undefined;
revenue?: number;
groups: string[];
}
type SelectHelper<T> = {
@@ -331,6 +335,7 @@ export async function getEvents(
projectId,
isExternal: false,
properties: {},
groups: [],
};
}
}
@@ -386,6 +391,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
sdk_name: payload.sdkName ?? '',
sdk_version: payload.sdkVersion ?? '',
revenue: payload.revenue,
groups: payload.groups ?? [],
};
const promises = [sessionBuffer.add(event), eventBuffer.add(event)];
@@ -434,6 +440,7 @@ export interface GetEventListOptions {
projectId: string;
profileId?: string;
sessionId?: string;
groupId?: string;
take: number;
cursor?: number | Date;
events?: string[] | null;
@@ -452,6 +459,7 @@ export async function getEventList(options: GetEventListOptions) {
projectId,
profileId,
sessionId,
groupId,
events,
filters,
startDate,
@@ -589,6 +597,10 @@ export async function getEventList(options: GetEventListOptions) {
sb.select.revenue = 'revenue';
}
if (select.groups) {
sb.select.groups = 'groups';
}
if (profileId) {
sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND device_id != '' AND profile_id = ${sqlstring.escape(profileId)} group by did) OR profile_id = ${sqlstring.escape(profileId)})`;
}
@@ -597,6 +609,10 @@ export async function getEventList(options: GetEventListOptions) {
sb.where.sessionId = `session_id = ${sqlstring.escape(sessionId)}`;
}
if (groupId) {
sb.where.groupId = `has(groups, ${sqlstring.escape(groupId)})`;
}
if (startDate && endDate) {
sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
}
@@ -611,7 +627,7 @@ export async function getEventList(options: GetEventListOptions) {
if (filters) {
sb.where = {
...sb.where,
...getEventFiltersWhereClause(filters),
...getEventFiltersWhereClause(filters, projectId),
};
// Join profiles table if any filter uses profile fields
@@ -622,6 +638,13 @@ export async function getEventList(options: GetEventListOptions) {
if (profileFilters.length > 0) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0])).join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
// Join groups table if any filter uses group fields
const groupFilters = filters.filter((f) => f.name.startsWith('group.'));
if (groupFilters.length > 0) {
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
}
}
sb.orderBy.created_at = 'created_at DESC, id ASC';
@@ -653,6 +676,7 @@ export async function getEventList(options: GetEventListOptions) {
export async function getEventsCount({
projectId,
profileId,
groupId,
events,
filters,
startDate,
@@ -664,6 +688,10 @@ export async function getEventsCount({
sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
}
if (groupId) {
sb.where.groupId = `has(groups, ${sqlstring.escape(groupId)})`;
}
if (startDate && endDate) {
sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
}
@@ -678,7 +706,7 @@ export async function getEventsCount({
if (filters) {
sb.where = {
...sb.where,
...getEventFiltersWhereClause(filters),
...getEventFiltersWhereClause(filters, projectId),
};
// Join profiles table if any filter uses profile fields
@@ -689,6 +717,13 @@ export async function getEventsCount({
if (profileFilters.length > 0) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0])).join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
// Join groups table if any filter uses group fields
const groupFilters = filters.filter((f) => f.name.startsWith('group.'));
if (groupFilters.length > 0) {
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
}
}
const res = await chQuery<{ count: number }>(
@@ -1052,8 +1087,19 @@ class EventService {
}
if (filters) {
q.rawWhere(
Object.values(getEventFiltersWhereClause(filters)).join(' AND ')
Object.values(
getEventFiltersWhereClause(filters, projectId)
).join(' AND ')
);
const groupFilters = filters.filter((f) =>
f.name.startsWith('group.')
);
if (groupFilters.length > 0) {
q.rawJoin('ARRAY JOIN groups AS _group_id');
q.rawJoin(
`LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`
);
}
}
},
session: (q) => {

33
packages/db/src/services/funnel.service.ts
View File

@@ -34,11 +34,11 @@ export class FunnelService {
return group === 'profile_id' ? 'profile_id' : 'session_id';
}
getFunnelConditions(events: IChartEvent[] = []): string[] {
getFunnelConditions(events: IChartEvent[] = [], projectId?: string): string[] {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.name = `events.name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
}
@@ -71,7 +71,7 @@ export class FunnelService {
additionalGroupBy?: string[];
group?: 'session_id' | 'profile_id';
}) {
const funnels = this.getFunnelConditions(eventSeries);
const funnels = this.getFunnelConditions(eventSeries, projectId);
const primaryKey = group === 'profile_id' ? 'profile_id' : 'session_id';
return clix(this.client, timezone)
@@ -90,7 +90,7 @@ export class FunnelService {
clix.datetime(endDate, 'toDateTime'),
])
.where(
'name',
'events.name',
'IN',
eventSeries.map((e) => e.name),
)
@@ -236,10 +236,18 @@ export class FunnelService {
const anyBreakdownOnProfile = breakdowns.some((b) =>
b.name.startsWith('profile.'),
);
const anyFilterOnGroup = eventSeries.some((e) =>
e.filters?.some((f) => f.name.startsWith('group.')),
);
const anyBreakdownOnGroup = breakdowns.some((b) =>
b.name.startsWith('group.'),
);
const needsGroupArrayJoin =
anyFilterOnGroup || anyBreakdownOnGroup || funnelGroup === 'group';
// Create the funnel CTE (session-level)
const breakdownSelects = breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
(b, index) => `${getSelectPropertyKey(b.name, projectId)} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
@@ -277,8 +285,21 @@ export class FunnelService {
);
}
if (needsGroupArrayJoin) {
funnelCte.rawJoin('ARRAY JOIN groups AS _group_id');
funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');
}
// Base funnel query with CTEs
const funnelQuery = clix(this.client, timezone);
if (needsGroupArrayJoin) {
funnelQuery.with(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`,
);
}
funnelQuery.with('session_funnel', funnelCte);
// windowFunnel is computed per the primary key (profile_id or session_id),

363
packages/db/src/services/group.service.ts Normal file
View File

@@ -0,0 +1,363 @@
import { toDots } from '@openpanel/common';
import sqlstring from 'sqlstring';
import {
ch,
chQuery,
formatClickhouseDate,
TABLE_NAMES,
} from '../clickhouse/client';
import type { IServiceProfile } from './profile.service';
import { getProfiles } from './profile.service';
export type IServiceGroup = {
id: string;
projectId: string;
type: string;
name: string;
properties: Record<string, unknown>;
createdAt: Date;
updatedAt: Date;
};
export type IServiceUpsertGroup = {
id: string;
projectId: string;
type: string;
name: string;
properties?: Record<string, unknown>;
};
type IClickhouseGroup = {
project_id: string;
id: string;
type: string;
name: string;
properties: Record<string, string>;
created_at: string;
version: string;
};
function transformGroup(row: IClickhouseGroup): IServiceGroup {
return {
id: row.id,
projectId: row.project_id,
type: row.type,
name: row.name,
properties: row.properties,
createdAt: new Date(row.created_at),
updatedAt: new Date(Number(row.version)),
};
}
async function writeGroupToCh(
group: {
id: string;
projectId: string;
type: string;
name: string;
properties: Record<string, string>;
createdAt?: Date;
},
deleted = 0
) {
await ch.insert({
format: 'JSONEachRow',
table: TABLE_NAMES.groups,
values: [
{
project_id: group.projectId,
id: group.id,
type: group.type,
name: group.name,
properties: group.properties,
created_at: formatClickhouseDate(group.createdAt ?? new Date()),
version: Date.now(),
deleted,
},
],
});
}
export async function upsertGroup(input: IServiceUpsertGroup) {
const existing = await getGroupById(input.id, input.projectId);
await writeGroupToCh({
id: input.id,
projectId: input.projectId,
type: input.type,
name: input.name,
properties: toDots({
...(existing?.properties ?? {}),
...(input.properties ?? {}),
}),
createdAt: existing?.createdAt,
});
}
export async function getGroupById(
id: string,
projectId: string
): Promise<IServiceGroup | null> {
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id = ${sqlstring.escape(id)}
AND deleted = 0
`);
return rows[0] ? transformGroup(rows[0]) : null;
}
export async function getGroupList({
projectId,
cursor,
take,
search,
type,
}: {
projectId: string;
cursor?: number;
take: number;
search?: string;
type?: string;
}): Promise<IServiceGroup[]> {
const conditions = [
`project_id = ${sqlstring.escape(projectId)}`,
'deleted = 0',
...(type ? [`type = ${sqlstring.escape(type)}`] : []),
...(search
? [
`(name ILIKE ${sqlstring.escape(`%${search}%`)} OR id ILIKE ${sqlstring.escape(`%${search}%`)})`,
]
: []),
];
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE ${conditions.join(' AND ')}
ORDER BY created_at DESC
LIMIT ${take}
OFFSET ${Math.max(0, (cursor ?? 0) * take)}
`);
return rows.map(transformGroup);
}
export async function getGroupListCount({
projectId,
type,
search,
}: {
projectId: string;
type?: string;
search?: string;
}): Promise<number> {
const conditions = [
`project_id = ${sqlstring.escape(projectId)}`,
'deleted = 0',
...(type ? [`type = ${sqlstring.escape(type)}`] : []),
...(search
? [
`(name ILIKE ${sqlstring.escape(`%${search}%`)} OR id ILIKE ${sqlstring.escape(`%${search}%`)})`,
]
: []),
];
const rows = await chQuery<{ count: number }>(`
SELECT count() as count
FROM ${TABLE_NAMES.groups} FINAL
WHERE ${conditions.join(' AND ')}
`);
return rows[0]?.count ?? 0;
}
export async function getGroupTypes(projectId: string): Promise<string[]> {
const rows = await chQuery<{ type: string }>(`
SELECT DISTINCT type
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND deleted = 0
`);
return rows.map((r) => r.type);
}
export async function createGroup(input: IServiceUpsertGroup) {
await upsertGroup(input);
return getGroupById(input.id, input.projectId);
}
export async function updateGroup(
id: string,
projectId: string,
data: { type?: string; name?: string; properties?: Record<string, unknown> }
) {
const existing = await getGroupById(id, projectId);
if (!existing) {
throw new Error(`Group ${id} not found`);
}
const mergedProperties = {
...(existing.properties ?? {}),
...(data.properties ?? {}),
};
const normalizedProperties = toDots(
mergedProperties as Record<string, unknown>
);
const updated = {
id,
projectId,
type: data.type ?? existing.type,
name: data.name ?? existing.name,
properties: normalizedProperties,
createdAt: existing.createdAt,
};
await writeGroupToCh(updated);
return { ...existing, ...updated };
}
export async function deleteGroup(id: string, projectId: string) {
const existing = await getGroupById(id, projectId);
if (!existing) {
throw new Error(`Group ${id} not found`);
}
await writeGroupToCh(
{
id,
projectId,
type: existing.type,
name: existing.name,
properties: existing.properties as Record<string, string>,
createdAt: existing.createdAt,
},
1
);
return existing;
}
export async function getGroupPropertyKeys(
projectId: string
): Promise<string[]> {
const rows = await chQuery<{ key: string }>(`
SELECT DISTINCT arrayJoin(mapKeys(properties)) as key
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND deleted = 0
`);
return rows.map((r) => r.key).sort();
}
export type IServiceGroupStats = {
groupId: string;
memberCount: number;
lastActiveAt: Date | null;
};
export async function getGroupStats(
projectId: string,
groupIds: string[]
): Promise<Map<string, IServiceGroupStats>> {
if (groupIds.length === 0) {
return new Map();
}
const rows = await chQuery<{
group_id: string;
member_count: number;
last_active_at: string;
}>(`
SELECT
g AS group_id,
uniqExact(profile_id) AS member_count,
max(created_at) AS last_active_at
FROM ${TABLE_NAMES.events}
ARRAY JOIN groups AS g
WHERE project_id = ${sqlstring.escape(projectId)}
AND g IN (${groupIds.map((id) => sqlstring.escape(id)).join(',')})
AND profile_id != device_id
GROUP BY g
`);
return new Map(
rows.map((r) => [
r.group_id,
{
groupId: r.group_id,
memberCount: r.member_count,
lastActiveAt: r.last_active_at ? new Date(r.last_active_at) : null,
},
])
);
}
export async function getGroupsByIds(
projectId: string,
ids: string[]
): Promise<IServiceGroup[]> {
if (ids.length === 0) {
return [];
}
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id IN (${ids.map((id) => sqlstring.escape(id)).join(',')})
AND deleted = 0
`);
return rows.map(transformGroup);
}
export async function getGroupMemberProfiles({
projectId,
groupId,
cursor,
take,
search,
}: {
projectId: string;
groupId: string;
cursor?: number;
take: number;
search?: string;
}): Promise<{ data: IServiceProfile[]; count: number }> {
const offset = Math.max(0, (cursor ?? 0) * take);
const searchCondition = search?.trim()
? `AND (email ILIKE ${sqlstring.escape(`%${search.trim()}%`)} OR first_name ILIKE ${sqlstring.escape(`%${search.trim()}%`)} OR last_name ILIKE ${sqlstring.escape(`%${search.trim()}%`)})`
: '';
// count() OVER () is evaluated after JOINs/WHERE but before LIMIT,
// so we get the total match count and the paginated IDs in one query.
const rows = await chQuery<{ profile_id: string; total_count: number }>(`
SELECT
gm.profile_id,
count() OVER () AS total_count
FROM (
SELECT profile_id, max(created_at) AS last_seen
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(groupId)})
AND profile_id != device_id
GROUP BY profile_id
) gm
INNER JOIN (
SELECT id FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
${searchCondition}
) p ON p.id = gm.profile_id
ORDER BY gm.last_seen DESC
LIMIT ${take}
OFFSET ${offset}
`);
const count = rows[0]?.total_count ?? 0;
const profileIds = rows.map((r) => r.profile_id);
if (profileIds.length === 0) {
return { data: [], count };
}
const profiles = await getProfiles(profileIds, projectId);
const byId = new Map(profiles.map((p) => [p.id, p]));
const data = profileIds
.map((id) => byId.get(id))
.filter(Boolean) as IServiceProfile[];
return { data, count };
}

2
packages/db/src/services/import.service.ts
View File

@@ -355,6 +355,7 @@ export async function createSessionsStartEndEvents(
profile_id: session.profile_id,
project_id: session.project_id,
session_id: session.session_id,
groups: [],
path: firstPath,
origin: firstOrigin,
referrer: firstReferrer,
@@ -390,6 +391,7 @@ export async function createSessionsStartEndEvents(
profile_id: session.profile_id,
project_id: session.project_id,
session_id: session.session_id,
groups: [],
path: lastPath,
origin: lastOrigin,
referrer: firstReferrer,

9
packages/db/src/services/profile.service.ts
View File

@@ -165,7 +165,8 @@ export async function getProfiles(ids: string[], projectId: string) {
any(nullIf(avatar, '')) as avatar,
last_value(is_external) as is_external,
any(properties) as properties,
any(created_at) as created_at
any(created_at) as created_at,
any(groups) as groups
FROM ${TABLE_NAMES.profiles}
WHERE
project_id = ${sqlstring.escape(projectId)} AND
@@ -232,6 +233,7 @@ export interface IServiceProfile {
createdAt: Date;
isExternal: boolean;
projectId: string;
groups: string[];
properties: Record<string, unknown> & {
region?: string;
country?: string;
@@ -259,6 +261,7 @@ export interface IClickhouseProfile {
project_id: string;
is_external: boolean;
created_at: string;
groups: string[];
}
export interface IServiceUpsertProfile {
@@ -270,6 +273,7 @@ export interface IServiceUpsertProfile {
avatar?: string;
properties?: Record<string, unknown>;
isExternal: boolean;
groups?: string[];
}
export function transformProfile({
@@ -288,6 +292,7 @@ export function transformProfile({
id: profile.id,
email: profile.email,
avatar: profile.avatar,
groups: profile.groups ?? [],
};
}
@@ -301,6 +306,7 @@ export function upsertProfile(
properties,
projectId,
isExternal,
groups,
}: IServiceUpsertProfile,
isFromEvent = false
) {
@@ -314,6 +320,7 @@ export function upsertProfile(
project_id: projectId,
created_at: formatClickhouseDate(new Date()),
is_external: isExternal,
groups: groups ?? [],
};
return profileBuffer.add(profile, isFromEvent);

5
packages/db/src/services/session.service.ts
View File

@@ -55,6 +55,7 @@ export interface IClickhouseSession {
version: number;
// Dynamically added
has_replay?: boolean;
groups: string[];
}
export interface IServiceSession {
@@ -95,6 +96,7 @@ export interface IServiceSession {
revenue: number;
profile?: IServiceProfile;
hasReplay?: boolean;
groups: string[];
}
export interface GetSessionListOptions {
@@ -152,6 +154,7 @@ export function transformSession(session: IClickhouseSession): IServiceSession {
revenue: session.revenue,
profile: undefined,
hasReplay: session.has_replay,
groups: session.groups,
};
}
@@ -244,6 +247,7 @@ export async function getSessionList(options: GetSessionListOptions) {
'screen_view_count',
'event_count',
'revenue',
'groups',
];
columns.forEach((column) => {
@@ -292,6 +296,7 @@ export async function getSessionList(options: GetSessionListOptions) {
projectId,
isExternal: false,
properties: {},
groups: [],
},
}));