zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: group analytics

Browse Source 
* wip

* wip

* wip

* wip

* wip

* add buffer

* wip

* wip

* fixes

* fix

* wip

* group validation

* fix group issues

* docs: add groups
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-20 10:46:09 +01:00
committed by GitHub
parent 88a2d876ce
commit 11e9ecac1a
99 changed files with 5944 additions and 1432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
packages/db/code-migrations/11-add-groups.ts Normal file
View File

@@ -0,0 +1,66 @@
import fs from 'node:fs';
import path from 'node:path';
import { TABLE_NAMES } from '../src/clickhouse/client';
import {
addColumns,
createTable,
runClickhouseMigrationCommands,
} from '../src/clickhouse/migration';
import { getIsCluster } from './helpers';
export async function up() {
const isClustered = getIsCluster();
const sqls: string[] = [
...addColumns(
'events',
['`groups` Array(String) DEFAULT [] CODEC(ZSTD(3)) AFTER session_id'],
isClustered
),
...addColumns(
'sessions',
['`groups` Array(String) DEFAULT [] CODEC(ZSTD(3)) AFTER device_id'],
isClustered
),
...addColumns(
'profiles',
['`groups` Array(String) DEFAULT [] CODEC(ZSTD(3)) AFTER project_id'],
isClustered
),
...createTable({
name: TABLE_NAMES.groups,
columns: [
'`id` String',
'`project_id` String',
'`type` String',
'`name` String',
'`properties` Map(String, String)',
'`created_at` DateTime',
'`version` UInt64',
'`deleted` UInt8 DEFAULT 0',
],
engine: 'ReplacingMergeTree(version, deleted)',
orderBy: ['project_id', 'id'],
distributionHash: 'cityHash64(project_id, id)',
replicatedVersion: '1',
isClustered,
}),
];
fs.writeFileSync(
path.join(import.meta.filename.replace('.ts', '.sql')),
sqls
.map((sql) =>
sql
.trim()
.replace(/;$/, '')
.replace(/\n{2,}/g, '\n')
.concat(';')
)
.join('\n\n---\n\n')
);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}

60
packages/db/index.ts
View File

@@ -1,35 +1,37 @@
export * from './src/prisma-client';
export * from './src/clickhouse/client';
export * from './src/sql-builder';
export * from './src/services/chart.service';
export * from './src/engine';
export * from './src/services/clients.service';
export * from './src/services/dashboard.service';
export * from './src/services/event.service';
export * from './src/services/organization.service';
export * from './src/services/profile.service';
export * from './src/services/project.service';
export * from './src/services/reports.service';
export * from './src/services/salt.service';
export * from './src/services/share.service';
export * from './src/services/session.service';
export * from './src/services/funnel.service';
export * from './src/services/conversion.service';
export * from './src/services/sankey.service';
export * from './src/services/user.service';
export * from './src/services/reference.service';
export * from './src/services/id.service';
export * from './src/services/retention.service';
export * from './src/services/notification.service';
export * from './src/services/access.service';
export * from './src/services/delete.service';
export * from './src/buffers';
export * from './src/types';
export * from './src/clickhouse/client';
export * from './src/clickhouse/query-builder';
export * from './src/encryption';
export * from './src/engine';
export * from './src/engine';
export * from './src/gsc';
export * from './src/prisma-client';
export * from './src/services/access.service';
export * from './src/services/chart.service';
export * from './src/services/clients.service';
export * from './src/services/conversion.service';
export * from './src/services/dashboard.service';
export * from './src/services/delete.service';
export * from './src/services/event.service';
export * from './src/services/funnel.service';
export * from './src/services/group.service';
export * from './src/services/id.service';
export * from './src/services/import.service';
export * from './src/services/insights';
export * from './src/services/notification.service';
export * from './src/services/organization.service';
export * from './src/services/overview.service';
export * from './src/services/pages.service';
export * from './src/services/insights';
export * from './src/services/profile.service';
export * from './src/services/project.service';
export * from './src/services/reference.service';
export * from './src/services/reports.service';
export * from './src/services/retention.service';
export * from './src/services/salt.service';
export * from './src/services/sankey.service';
export * from './src/services/session.service';
export * from './src/services/share.service';
export * from './src/services/user.service';
export * from './src/session-context';
export * from './src/gsc';
export * from './src/encryption';
export * from './src/sql-builder';
export * from './src/types';

2
packages/db/prisma/schema.prisma
View File

@@ -199,7 +199,6 @@ model Project {
meta EventMeta[]
references Reference[]
access ProjectAccess[]
notificationRules NotificationRule[]
notifications Notification[]
imports Import[]
@@ -215,6 +214,7 @@ model Project {
@@map("projects")
}
enum AccessLevel {
read
write

23
packages/db/src/buffers/event-buffer.ts
View File

@@ -1,11 +1,7 @@
import { getSafeJson } from '@openpanel/json';
import {
type Redis,
getRedisCache,
publishEvent,
} from '@openpanel/redis';
import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import { type IClickhouseEvent } from '../services/event.service';
import type { IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
export class EventBuffer extends BaseBuffer {
@@ -95,7 +91,7 @@ export class EventBuffer extends BaseBuffer {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id,
event.profile_id
);
}
}
@@ -116,7 +112,7 @@ export class EventBuffer extends BaseBuffer {
error,
eventCount: eventsToFlush.length,
flushRetryCount: this.flushRetryCount,
},
}
);
} finally {
this.isFlushing = false;
@@ -137,7 +133,7 @@ export class EventBuffer extends BaseBuffer {
const queueEvents = await redis.lrange(
this.queueKey,
0,
this.batchSize - 1,
this.batchSize - 1
);
if (queueEvents.length === 0) {
@@ -149,6 +145,9 @@ export class EventBuffer extends BaseBuffer {
for (const eventStr of queueEvents) {
const event = getSafeJson<IClickhouseEvent>(eventStr);
if (event) {
if (!Array.isArray(event.groups)) {
event.groups = [];
}
eventsToClickhouse.push(event);
}
}
@@ -161,7 +160,7 @@ export class EventBuffer extends BaseBuffer {
eventsToClickhouse.sort(
(a, b) =>
new Date(a.created_at || 0).getTime() -
new Date(b.created_at || 0).getTime(),
new Date(b.created_at || 0).getTime()
);
this.logger.info('Inserting events into ClickHouse', {
@@ -181,7 +180,7 @@ export class EventBuffer extends BaseBuffer {
for (const event of eventsToClickhouse) {
countByProject.set(
event.project_id,
(countByProject.get(event.project_id) ?? 0) + 1,
(countByProject.get(event.project_id) ?? 0) + 1
);
}
for (const [projectId, count] of countByProject) {
@@ -222,7 +221,7 @@ export class EventBuffer extends BaseBuffer {
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string,
profileId: string
) {
const key = `${projectId}:${profileId}`;
const now = Date.now();

195
packages/db/src/buffers/group-buffer.ts Normal file
View File

@@ -0,0 +1,195 @@
import { toDots } from '@openpanel/common';
import { getSafeJson } from '@openpanel/json';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import sqlstring from 'sqlstring';
import { ch, chQuery, formatClickhouseDate, TABLE_NAMES } from '../clickhouse/client';
import { BaseBuffer } from './base-buffer';
type IGroupBufferEntry = {
project_id: string;
id: string;
type: string;
name: string;
properties: Record<string, string>;
created_at: string;
version: string;
deleted: number;
};
type IGroupCacheEntry = {
id: string;
project_id: string;
type: string;
name: string;
properties: Record<string, string>;
created_at: string;
};
export type IGroupBufferInput = {
id: string;
projectId: string;
type: string;
name: string;
properties?: Record<string, unknown>;
};
export class GroupBuffer extends BaseBuffer {
private batchSize = process.env.GROUP_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.GROUP_BUFFER_BATCH_SIZE, 10)
: 200;
private chunkSize = process.env.GROUP_BUFFER_CHUNK_SIZE
? Number.parseInt(process.env.GROUP_BUFFER_CHUNK_SIZE, 10)
: 1000;
private ttlInSeconds = process.env.GROUP_BUFFER_TTL_IN_SECONDS
? Number.parseInt(process.env.GROUP_BUFFER_TTL_IN_SECONDS, 10)
: 60 * 60;
private readonly redisKey = 'group-buffer';
private readonly redisCachePrefix = 'group-cache:';
private redis: Redis;
constructor() {
super({
name: 'group',
onFlush: async () => {
await this.processBuffer();
},
});
this.redis = getRedisCache();
}
private getCacheKey(projectId: string, id: string) {
return `${this.redisCachePrefix}${projectId}:${id}`;
}
private async fetchFromCache(
projectId: string,
id: string
): Promise<IGroupCacheEntry | null> {
const raw = await this.redis.get(this.getCacheKey(projectId, id));
if (!raw) return null;
return getSafeJson<IGroupCacheEntry>(raw);
}
private async fetchFromClickhouse(
projectId: string,
id: string
): Promise<IGroupCacheEntry | null> {
const rows = await chQuery<IGroupCacheEntry>(`
SELECT project_id, id, type, name, properties, created_at
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id = ${sqlstring.escape(id)}
AND deleted = 0
`);
return rows[0] ?? null;
}
async add(input: IGroupBufferInput): Promise<void> {
try {
const cacheKey = this.getCacheKey(input.projectId, input.id);
const existing =
(await this.fetchFromCache(input.projectId, input.id)) ??
(await this.fetchFromClickhouse(input.projectId, input.id));
const mergedProperties = toDots({
...(existing?.properties ?? {}),
...(input.properties ?? {}),
}) as Record<string, string>;
const entry: IGroupBufferEntry = {
project_id: input.projectId,
id: input.id,
type: input.type,
name: input.name,
properties: mergedProperties,
created_at: formatClickhouseDate(
existing?.created_at ? new Date(existing.created_at) : new Date()
),
version: String(Date.now()),
deleted: 0,
};
if (
existing &&
existing.type === entry.type &&
existing.name === entry.name &&
shallowEqual(existing.properties, entry.properties)
) {
this.logger.debug('Group not changed, skipping', { id: input.id });
return;
}
const cacheEntry: IGroupCacheEntry = {
id: entry.id,
project_id: entry.project_id,
type: entry.type,
name: entry.name,
properties: entry.properties,
created_at: entry.created_at,
};
const result = await this.redis
.multi()
.set(cacheKey, JSON.stringify(cacheEntry), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(entry))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add group to Redis', { input });
return;
}
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
} catch (error) {
this.logger.error('Failed to add group', { error, input });
}
}
async processBuffer(): Promise<void> {
try {
this.logger.debug('Starting group buffer processing');
const items = await this.redis.lrange(this.redisKey, 0, this.batchSize - 1);
if (items.length === 0) {
this.logger.debug('No groups to process');
return;
}
this.logger.debug(`Processing ${items.length} groups in buffer`);
const parsed = items.map((i) => getSafeJson<IGroupBufferEntry>(i));
for (const chunk of this.chunks(parsed, this.chunkSize)) {
await ch.insert({
table: TABLE_NAMES.groups,
values: chunk,
format: 'JSONEachRow',
});
}
await this.redis
.multi()
.ltrim(this.redisKey, items.length, -1)
.decrby(this.bufferCounterKey, items.length)
.exec();
this.logger.debug('Successfully completed group processing', {
totalGroups: items.length,
});
} catch (error) {
this.logger.error('Failed to process buffer', { error });
}
}
async getBufferSize(): Promise<number> {
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
}
}

2
packages/db/src/buffers/index.ts
View File

@@ -1,5 +1,6 @@
import { BotBuffer as BotBufferRedis } from './bot-buffer';
import { EventBuffer as EventBufferRedis } from './event-buffer';
import { GroupBuffer } from './group-buffer';
import { ProfileBackfillBuffer } from './profile-backfill-buffer';
import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
import { ReplayBuffer } from './replay-buffer';
@@ -11,6 +12,7 @@ export const botBuffer = new BotBufferRedis();
export const sessionBuffer = new SessionBuffer();
export const profileBackfillBuffer = new ProfileBackfillBuffer();
export const replayBuffer = new ReplayBuffer();
export const groupBuffer = new GroupBuffer();
export type { ProfileBackfillEntry } from './profile-backfill-buffer';
export type { IClickhouseSessionReplayChunk } from './replay-buffer';

151
packages/db/src/buffers/profile-buffer.test.ts Normal file
View File

@@ -0,0 +1,151 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { getSafeJson } from '@openpanel/json';
import type { IClickhouseProfile } from '../services/profile.service';
// Mock chQuery to avoid hitting real ClickHouse
vi.mock('../clickhouse/client', () => ({
ch: {
insert: vi.fn().mockResolvedValue(undefined),
},
chQuery: vi.fn().mockResolvedValue([]),
TABLE_NAMES: {
profiles: 'profiles',
},
}));
import { ProfileBuffer } from './profile-buffer';
import { chQuery } from '../clickhouse/client';
const redis = getRedisCache();
function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile {
return {
id: 'profile-1',
project_id: 'project-1',
first_name: '',
last_name: '',
email: '',
avatar: '',
properties: {},
is_external: true,
created_at: new Date().toISOString(),
groups: [],
...overrides,
};
}
beforeEach(async () => {
await redis.flushdb();
vi.mocked(chQuery).mockResolvedValue([]);
});
afterAll(async () => {
try {
await redis.quit();
} catch {}
});
describe('ProfileBuffer', () => {
let profileBuffer: ProfileBuffer;
beforeEach(() => {
profileBuffer = new ProfileBuffer();
});
it('adds a profile to the buffer', async () => {
const profile = makeProfile({ first_name: 'John', email: 'john@example.com' });
const sizeBefore = await profileBuffer.getBufferSize();
await profileBuffer.add(profile);
const sizeAfter = await profileBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore + 1);
});
it('merges subsequent updates via cache (sequential calls)', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Sequential: identify first, then group
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
// Second add should read the cached identify profile and merge groups in
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent identify + group calls preserve all data', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Both calls run concurrently — the per-profile lock serializes them so the
// second one reads the first's result from cache and merges correctly.
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent writes produce one merged buffer entry', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
const sizeBefore = await profileBuffer.getBufferSize();
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const sizeAfter = await profileBuffer.getBufferSize();
// The second add merges into the first — only 2 buffer entries total
// (one from identify, one merged update with group)
expect(sizeAfter).toBe(sizeBefore + 2);
// The last entry in the buffer should have both name and group
const rawEntries = await redis.lrange('profile-buffer', 0, -1);
const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e));
const lastEntry = entries[entries.length - 1];
expect(lastEntry?.first_name).toBe('John');
expect(lastEntry?.groups).toContain('group-abc');
});
});

200
packages/db/src/buffers/profile-buffer.ts
View File

@@ -1,9 +1,10 @@
import { deepMergeObjects } from '@openpanel/common';
import { generateSecureId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json';
import type { ILogger } from '@openpanel/logger';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import { omit } from 'ramda';
import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
import type { IClickhouseProfile } from '../services/profile.service';
@@ -24,6 +25,15 @@ export class ProfileBuffer extends BaseBuffer {
private readonly redisProfilePrefix = 'profile-cache:';
private redis: Redis;
private releaseLockSha: string | null = null;
private readonly releaseLockScript = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
else
return 0
end
`;
constructor() {
super({
@@ -33,6 +43,9 @@ export class ProfileBuffer extends BaseBuffer {
},
});
this.redis = getRedisCache();
this.redis.script('LOAD', this.releaseLockScript).then((sha) => {
this.releaseLockSha = sha as string;
});
}
private getProfileCacheKey({
@@ -45,6 +58,42 @@ export class ProfileBuffer extends BaseBuffer {
return `${this.redisProfilePrefix}${projectId}:${profileId}`;
}
private async withProfileLock<T>(
profileId: string,
projectId: string,
fn: () => Promise<T>
): Promise<T> {
const lockKey = `profile-lock:${projectId}:${profileId}`;
const lockId = generateSecureId('lock');
const maxRetries = 20;
const retryDelayMs = 50;
for (let i = 0; i < maxRetries; i++) {
const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');
if (acquired === 'OK') {
try {
return await fn();
} finally {
if (this.releaseLockSha) {
await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);
} else {
await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);
}
}
}
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
}
this.logger.error(
'Failed to acquire profile lock, proceeding without lock',
{
profileId,
projectId,
}
);
return fn();
}
async alreadyExists(profile: IClickhouseProfile) {
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
@@ -67,83 +116,94 @@ export class ProfileBuffer extends BaseBuffer {
return;
}
const existingProfile = await this.fetchProfile(profile, logger);
await this.withProfileLock(profile.id, profile.project_id, async () => {
const existingProfile = await this.fetchProfile(profile, logger);
// Delete any properties that are not server related if we have a non-server profile
if (
existingProfile?.properties.device !== 'server' &&
profile.properties.device === 'server'
) {
profile.properties = omit(
[
'city',
'country',
'region',
'longitude',
'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
profile.properties
);
}
// Delete any properties that are not server related if we have a non-server profile
if (
existingProfile?.properties.device !== 'server' &&
profile.properties.device === 'server'
) {
profile.properties = omit(
[
'city',
'country',
'region',
'longitude',
'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
profile.properties
);
}
const mergedProfile: IClickhouseProfile = existingProfile
? deepMergeObjects(existingProfile, omit(['created_at'], profile))
: profile;
const mergedProfile: IClickhouseProfile = existingProfile
? {
...deepMergeObjects(
existingProfile,
omit(['created_at', 'groups'], profile)
),
groups: uniq([
...(existingProfile.groups ?? []),
...(profile.groups ?? []),
]),
}
: profile;
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
this.logger.debug('Merged profile will be inserted', {
mergedProfile,
existingProfile,
profile,
});
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
const result = await this.redis
.multi()
.set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(mergedProfile))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add profile to Redis', {
this.logger.debug('Merged profile will be inserted', {
mergedProfile,
existingProfile,
profile,
cacheKey,
});
return;
}
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
this.logger.debug('Current buffer length', {
bufferLength,
batchSize: this.batchSize,
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
const result = await this.redis
.multi()
.set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(mergedProfile))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add profile to Redis', {
profile,
cacheKey,
});
return;
}
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
this.logger.debug('Current buffer length', {
bufferLength,
batchSize: this.batchSize,
});
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
});
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
} catch (error) {
this.logger.error('Failed to add profile', { error, profile });
}

7
packages/db/src/buffers/session-buffer.ts
View File

@@ -109,6 +109,12 @@ export class SessionBuffer extends BaseBuffer {
newSession.profile_id = event.profile_id;
}
if (event.groups) {
newSession.groups = [
...new Set([...newSession.groups, ...event.groups]),
];
}
return [newSession, oldSession];
}
@@ -119,6 +125,7 @@ export class SessionBuffer extends BaseBuffer {
profile_id: event.profile_id,
project_id: event.project_id,
device_id: event.device_id,
groups: event.groups,
created_at: event.created_at,
ended_at: event.created_at,
event_count: event.name === 'screen_view' ? 0 : 1,

9
packages/db/src/clickhouse/client.ts
View File

@@ -61,6 +61,7 @@ export const TABLE_NAMES = {
gsc_daily: 'gsc_daily',
gsc_pages_daily: 'gsc_pages_daily',
gsc_queries_daily: 'gsc_queries_daily',
groups: 'groups',
};
/**
@@ -298,3 +299,11 @@ const ROLLUP_DATE_PREFIX = '1970-01-01';
export function isClickhouseDefaultMinDate(date: string): boolean {
return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');
}
export function toNullIfDefaultMinDate(date?: string | null): Date | null {
if (!date) {
return null;
}
return isClickhouseDefaultMinDate(date)
? null
: convertClickhouseDateToJs(date);
}

12
packages/db/src/clickhouse/query-builder.ts
View File

@@ -66,6 +66,7 @@ export class Query<T = any> {
alias?: string;
}[] = [];
private _skipNext = false;
private _rawJoins: string[] = [];
private _fill?: {
from: string | Date;
to: string | Date;
@@ -339,6 +340,12 @@ export class Query<T = any> {
return this.joinWithType('CROSS', table, '', alias);
}
rawJoin(sql: string): this {
if (this._skipNext) return this;
this._rawJoins.push(sql);
return this;
}
private joinWithType(
type: JoinType,
table: string | Expression | Query,
@@ -426,6 +433,10 @@ export class Query<T = any> {
`${join.type} JOIN ${join.table instanceof Query ? `(${join.table.toSQL()})` : join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`
);
});
// Add raw joins (e.g. ARRAY JOIN)
this._rawJoins.forEach((join) => {
parts.push(join);
});
}
// WHERE
@@ -604,6 +615,7 @@ export class Query<T = any> {
// Merge JOINS
this._joins = [...this._joins, ...query._joins];
this._rawJoins = [...this._rawJoins, ...query._rawJoins];
// Merge settings
this._settings = { ...this._settings, ...query._settings };

16
packages/db/src/engine/format.ts
View File

@@ -26,7 +26,7 @@ export function format(
}>,
includeAlphaIds: boolean,
previousSeries: ConcreteSeries[] | null = null,
limit: number | undefined = undefined,
limit: number | undefined = undefined
): FinalChart {
const series = concreteSeries.map((cs) => {
// Find definition for this series
@@ -70,7 +70,7 @@ export function format(
const previousSerie = previousSeries?.find(
(ps) =>
ps.definitionIndex === cs.definitionIndex &&
ps.name.slice(1).join(':::') === cs.name.slice(1).join(':::'),
ps.name.slice(1).join(':::') === cs.name.slice(1).join(':::')
);
return {
@@ -89,24 +89,24 @@ export function format(
previous: {
sum: getPreviousMetric(
metrics.sum,
sum(previousSerie.data.map((d) => d.count)),
sum(previousSerie.data.map((d) => d.count))
),
average: getPreviousMetric(
metrics.average,
round(average(previousSerie.data.map((d) => d.count)), 2),
round(average(previousSerie.data.map((d) => d.count)), 2)
),
min: getPreviousMetric(
metrics.min,
min(previousSerie.data.map((d) => d.count)),
min(previousSerie.data.map((d) => d.count))
),
max: getPreviousMetric(
metrics.max,
max(previousSerie.data.map((d) => d.count)),
max(previousSerie.data.map((d) => d.count))
),
count: getPreviousMetric(
metrics.count ?? 0,
previousSerie.data.find((item) => !!item.total_count)
?.total_count ?? null,
?.total_count ?? null
),
},
}
@@ -118,7 +118,7 @@ export function format(
previous: previousSerie?.data[index]
? getPreviousMetric(
item.count,
previousSerie.data[index]?.count ?? null,
previousSerie.data[index]?.count ?? null
)
: undefined,
})),

19
packages/db/src/engine/index.ts
View File

@@ -1,5 +1,5 @@
import { getPreviousMetric, groupByLabels } from '@openpanel/common';
import type { ISerieDataItem } from '@openpanel/common';
import { groupByLabels } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import type {
FinalChart,
@@ -33,7 +33,7 @@ export async function executeChart(input: IReportInput): Promise<FinalChart> {
// Handle subscription end date limit
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
normalized.endDate,
normalized.endDate
);
if (endDate) {
normalized.endDate = endDate;
@@ -73,6 +73,7 @@ export async function executeChart(input: IReportInput): Promise<FinalChart> {
executionPlan.definitions,
includeAlphaIds,
previousSeries,
normalized.limit
);
return response;
@@ -83,7 +84,7 @@ export async function executeChart(input: IReportInput): Promise<FinalChart> {
* Executes a simplified pipeline: normalize -> fetch aggregate -> format
*/
export async function executeAggregateChart(
input: IReportInput,
input: IReportInput
): Promise<FinalChart> {
// Stage 1: Normalize input
const normalized = await normalize(input);
@@ -91,7 +92,7 @@ export async function executeAggregateChart(
// Handle subscription end date limit
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
normalized.endDate,
normalized.endDate
);
if (endDate) {
normalized.endDate = endDate;
@@ -137,7 +138,7 @@ export async function executeAggregateChart(
getAggregateChartSql(queryInput),
{
session_timezone: timezone,
},
}
);
// Fallback: if no results with breakdowns, try without breakdowns
@@ -149,7 +150,7 @@ export async function executeAggregateChart(
}),
{
session_timezone: timezone,
},
}
);
}
@@ -262,7 +263,7 @@ export async function executeAggregateChart(
getAggregateChartSql(queryInput),
{
session_timezone: timezone,
},
}
);
if (queryResult.length === 0 && normalized.breakdowns.length > 0) {
@@ -273,7 +274,7 @@ export async function executeAggregateChart(
}),
{
session_timezone: timezone,
},
}
);
}
@@ -344,7 +345,7 @@ export async function executeAggregateChart(
normalized.series,
includeAlphaIds,
previousSeries,
normalized.limit,
normalized.limit
);
return response;

499
packages/db/src/services/chart.service.ts
View File

@@ -1,20 +1,19 @@
import sqlstring from 'sqlstring';
/** biome-ignore-all lint/style/useDefaultSwitchClause: switch cases are exhaustive by design */
import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common';
import type {
IChartEventFilter,
IReportInput,
IChartRange,
IGetChartDataInput,
IReportInput,
} from '@openpanel/validation';
import { TABLE_NAMES, formatClickhouseDate } from '../clickhouse/client';
import sqlstring from 'sqlstring';
import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
export function transformPropertyKey(property: string) {
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
property.startsWith(`${pattern}.`)
);
if (!match) {
@@ -32,21 +31,91 @@ export function transformPropertyKey(property: string) {
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
export function getSelectPropertyKey(property: string) {
// Returns a SQL expression for a group property via the _g JOIN alias
// property format: "group.name", "group.type", "group.properties.plan"
export function getGroupPropertySql(property: string): string {
const withoutPrefix = property.replace(/^group\./, '');
if (withoutPrefix === 'name') {
return '_g.name';
}
if (withoutPrefix === 'type') {
return '_g.type';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `_g.properties[${sqlstring.escape(propKey)}]`;
}
return '_group_id';
}
// Returns the SELECT expression when querying the groups table directly (no join alias).
// Use for fetching distinct values for group.* properties.
export function getGroupPropertySelect(property: string): string {
const withoutPrefix = property.replace(/^group\./, '');
if (withoutPrefix === 'name') {
return 'name';
}
if (withoutPrefix === 'type') {
return 'type';
}
if (withoutPrefix === 'id') {
return 'id';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `properties[${sqlstring.escape(propKey)}]`;
}
return 'id';
}
// Returns the SELECT expression when querying the profiles table directly (no join alias).
// Use for fetching distinct values for profile.* properties.
export function getProfilePropertySelect(property: string): string {
const withoutPrefix = property.replace(/^profile\./, '');
if (withoutPrefix === 'id') {
return 'id';
}
if (withoutPrefix === 'first_name') {
return 'first_name';
}
if (withoutPrefix === 'last_name') {
return 'last_name';
}
if (withoutPrefix === 'email') {
return 'email';
}
if (withoutPrefix === 'avatar') {
return 'avatar';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `properties[${sqlstring.escape(propKey)}]`;
}
return 'id';
}
export function getSelectPropertyKey(property: string, projectId?: string) {
if (property === 'has_profile') {
return `if(profile_id != device_id, 'true', 'false')`;
}
// Handle group properties — requires ARRAY JOIN + _g JOIN to be present in query
if (property.startsWith('group.') && projectId) {
return getGroupPropertySql(property);
}
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
property.startsWith(`${pattern}.`)
);
if (!match) return property;
if (!match) {
return property;
}
if (property.includes('*')) {
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${sqlstring.escape(
transformPropertyKey(property),
transformPropertyKey(property)
)})))`;
}
@@ -60,9 +129,7 @@ export function getChartSql({
startDate,
endDate,
projectId,
limit,
timezone,
chartType,
}: IGetChartDataInput & { timezone: string }) {
const {
sb,
@@ -78,22 +145,43 @@ export function getChartSql({
with: addCte,
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
sb.where.eventName = `e.name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
filter.name.startsWith('profile.'),
filter.name.startsWith('profile.')
);
const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
breakdown.name.startsWith('profile.'),
breakdown.name.startsWith('profile.')
);
const anyFilterOnGroup = event.filters.some((filter) =>
filter.name.startsWith('group.')
);
const anyBreakdownOnGroup = breakdowns.some((breakdown) =>
breakdown.name.startsWith('group.')
);
const anyMetricOnGroup = !!event.property?.startsWith('group.');
const needsGroupArrayJoin =
anyFilterOnGroup ||
anyBreakdownOnGroup ||
anyMetricOnGroup ||
event.segment === 'group';
if (needsGroupArrayJoin) {
addCte(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_table = 'LEFT ANY JOIN _g ON _g.id = _group_id';
}
// Build WHERE clause without the bar filter (for use in subqueries and CTEs)
// Define this early so we can use it in CTE definitions
@@ -178,8 +266,8 @@ export function getChartSql({
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`
);
// Use the CTE reference in the main query
@@ -225,31 +313,11 @@ export function getChartSql({
sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
}
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
.map((b) => getSelectPropertyKey(b.name))
.join(', ');
// Add top_breakdowns CTE using the builder
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.select[key] =
`${getSelectPropertyKey(breakdown.name, projectId)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -261,6 +329,10 @@ export function getChartSql({
sb.select.count = 'countDistinct(session_id) as count';
}
if (event.segment === 'group') {
sb.select.count = 'countDistinct(_group_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
@@ -287,9 +359,9 @@ export function getChartSql({
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} e ${getJoins()} WHERE ${join(
sb.where,
' AND ',
' AND '
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
@@ -303,41 +375,52 @@ export function getChartSql({
}
// Note: The profile CTE (if it exists) is available in subqueries, so we can reference it directly
const subqueryGroupJoins = needsGroupArrayJoin
? 'ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id '
: '';
if (breakdowns.length > 0) {
// Match breakdown properties in subquery with outer query's grouped values
// Since outer query groups by label_X, we reference those in the correlation
const breakdownMatches = breakdowns
// Pre-compute unique counts per breakdown group in a CTE, then JOIN it.
// We can't use a correlated subquery because:
// 1. ClickHouse expands label_X aliases to their underlying expressions,
// which resolve in the subquery's scope, making the condition a tautology.
// 2. Correlated subqueries aren't supported on distributed/remote tables.
const ucSelectParts: string[] = breakdowns.map((breakdown, index) => {
const propertyKey = getSelectPropertyKey(breakdown.name, projectId);
return `${propertyKey} as _uc_label_${index + 1}`;
});
ucSelectParts.push('uniq(profile_id) as total_count');
const ucGroupByParts = breakdowns.map(
(_, index) => `_uc_label_${index + 1}`
);
const ucWhere = getWhereWithoutBar();
addCte(
'_uc',
`SELECT ${ucSelectParts.join(', ')} FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${ucWhere} GROUP BY ${ucGroupByParts.join(', ')}`
);
const ucJoinConditions = breakdowns
.map((b, index) => {
const propertyKey = getSelectPropertyKey(b.name);
// Correlate: match the property expression with outer query's label_X value
// ClickHouse allows referencing outer query columns in correlated subqueries
return `${propertyKey} = label_${index + 1}`;
const propertyKey = getSelectPropertyKey(b.name, projectId);
return `_uc._uc_label_${index + 1} = ${propertyKey}`;
})
.join(' AND ');
// Build WHERE clause for subquery - replace table alias and keep profile CTE reference
const subqueryWhere = getWhereWithoutBar()
.replace(/\be\./g, 'e2.')
.replace(/\bprofile\./g, 'profile.');
sb.select.total_unique_count = `(
SELECT uniq(profile_id)
FROM ${TABLE_NAMES.events} e2
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
AND ${breakdownMatches}
) as total_count`;
sb.joins.unique_counts = `LEFT ANY JOIN _uc ON ${ucJoinConditions}`;
sb.select.total_unique_count = 'any(_uc.total_count) as total_count';
} else {
// No breakdowns: calculate unique count across all data
// Build WHERE clause for subquery - replace table alias and keep profile CTE reference
const subqueryWhere = getWhereWithoutBar()
.replace(/\be\./g, 'e2.')
.replace(/\bprofile\./g, 'profile.');
const ucWhere = getWhereWithoutBar();
sb.select.total_unique_count = `(
SELECT uniq(profile_id)
FROM ${TABLE_NAMES.events} e2
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
) as total_count`;
addCte(
'_uc',
`SELECT uniq(profile_id) as total_count FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${ucWhere}`
);
sb.select.total_unique_count =
'(SELECT total_count FROM _uc) as total_count';
}
const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
@@ -359,31 +442,43 @@ export function getAggregateChartSql({
}) {
const { sb, join, getJoins, with: addCte, getSql } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
sb.where.eventName = `e.name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
filter.name.startsWith('profile.'),
filter.name.startsWith('profile.')
);
const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
breakdown.name.startsWith('profile.'),
breakdown.name.startsWith('profile.')
);
const anyFilterOnGroup = event.filters.some((filter) =>
filter.name.startsWith('group.')
);
const anyBreakdownOnGroup = breakdowns.some((breakdown) =>
breakdown.name.startsWith('group.')
);
const anyMetricOnGroup = !!event.property?.startsWith('group.');
const needsGroupArrayJoin =
anyFilterOnGroup ||
anyBreakdownOnGroup ||
anyMetricOnGroup ||
event.segment === 'group';
// Build WHERE clause without the bar filter (for use in subqueries and CTEs)
const getWhereWithoutBar = () => {
const whereWithoutBar = { ...sb.where };
delete whereWithoutBar.bar;
return Object.keys(whereWithoutBar).length
? `WHERE ${join(whereWithoutBar, ' AND ')}`
: '';
};
if (needsGroupArrayJoin) {
addCte(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_table = 'LEFT ANY JOIN _g ON _g.id = _group_id';
}
// Collect all profile fields used in filters and breakdowns
const getProfileFields = () => {
@@ -455,8 +550,8 @@ export function getAggregateChartSql({
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.profiles = profilesJoinRef;
@@ -475,31 +570,12 @@ export function getAggregateChartSql({
// Use startDate as the date value since we're aggregating across the entire range
sb.select.date = `${sqlstring.escape(startDate)} as date`;
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
.map((b) => getSelectPropertyKey(b.name))
.join(', ');
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
// Add breakdowns to SELECT and GROUP BY
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.select[key] =
`${getSelectPropertyKey(breakdown.name, projectId)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -518,6 +594,10 @@ export function getAggregateChartSql({
sb.select.count = 'countDistinct(session_id) as count';
}
if (event.segment === 'group') {
sb.select.count = 'countDistinct(_group_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
@@ -531,7 +611,7 @@ export function getAggregateChartSql({
}[event.segment as string];
if (mathFunction && event.property) {
const propertyKey = getSelectPropertyKey(event.property);
const propertyKey = getSelectPropertyKey(event.property, projectId);
if (isNumericColumn(event.property)) {
sb.select.count = `${mathFunction}(${propertyKey}) as count`;
@@ -544,9 +624,9 @@ export function getAggregateChartSql({
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} e ${getJoins()} WHERE ${join(
sb.where,
' AND ',
' AND '
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
@@ -579,7 +659,10 @@ function isNumericColumn(columnName: string): boolean {
return numericColumns.includes(columnName);
}
export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
export function getEventFiltersWhereClause(
filters: IChartEventFilter[],
projectId?: string
) {
const where: Record<string, string> = {};
filters.forEach((filter, index) => {
const id = `f${index}`;
@@ -602,6 +685,67 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
return;
}
// Handle group. prefixed filters (requires ARRAY JOIN + _g JOIN in query)
if (name.startsWith('group.') && projectId) {
const whereFrom = getGroupPropertySql(name);
switch (operator) {
case 'is': {
if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] =
`${whereFrom} IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')})`;
}
break;
}
case 'isNot': {
if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] =
`${whereFrom} NOT IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')})`;
}
break;
}
case 'contains': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'doesNotContain': {
where[id] =
`(${value.map((val) => `${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'startsWith': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'endsWith': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`).join(' OR ')})`;
break;
}
case 'isNull': {
where[id] = `(${whereFrom} = '' OR ${whereFrom} IS NULL)`;
break;
}
case 'isNotNull': {
where[id] = `(${whereFrom} != '' AND ${whereFrom} IS NOT NULL)`;
break;
}
case 'regex': {
where[id] =
`(${value.map((val) => `match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`).join(' OR ')})`;
break;
}
}
return;
}
if (
name.startsWith('properties.') ||
name.startsWith('profile.properties.')
@@ -616,15 +760,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x = ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
where[id] = `${whereFrom} IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
}
@@ -633,15 +775,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x != ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} NOT IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
where[id] = `${whereFrom} NOT IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
}
@@ -649,15 +789,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -668,14 +807,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -685,14 +824,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
(val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -702,14 +841,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')})`;
}
@@ -724,7 +863,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`,
`match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -752,14 +891,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -770,14 +909,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -788,14 +927,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -806,14 +945,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -856,7 +995,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -865,7 +1004,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -874,7 +1013,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
`${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -883,7 +1022,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')})`;
break;
@@ -892,7 +1031,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
`match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`
)
.join(' OR ')})`;
break;
@@ -902,7 +1041,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
@@ -917,7 +1056,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
@@ -932,13 +1071,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
where[id] = `(${value
.map(
(val) => `${name} >= ${sqlstring.escape(String(val).trim())}`,
(val) => `${name} >= ${sqlstring.escape(String(val).trim())}`
)
.join(' OR ')})`;
}
@@ -949,13 +1088,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
where[id] = `(${value
.map(
(val) => `${name} <= ${sqlstring.escape(String(val).trim())}`,
(val) => `${name} <= ${sqlstring.escape(String(val).trim())}`
)
.join(' OR ')})`;
}
@@ -974,15 +1113,15 @@ export function getChartStartEndDate(
endDate,
range,
}: Pick<IReportInput, 'endDate' | 'startDate' | 'range'>,
timezone: string,
timezone: string
) {
if (startDate && endDate) {
return { startDate: startDate, endDate: endDate };
return { startDate, endDate };
}
const ranges = getDatesFromRange(range, timezone);
if (!startDate && endDate) {
return { startDate: ranges.startDate, endDate: endDate };
return { startDate: ranges.startDate, endDate };
}
return ranges;
@@ -1002,8 +1141,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1018,8 +1157,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1035,8 +1174,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.endOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1053,8 +1192,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1071,8 +1210,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1089,8 +1228,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1106,8 +1245,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1124,8 +1263,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1141,8 +1280,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1152,8 +1291,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1170,8 +1309,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1183,7 +1322,7 @@ export function getChartPrevStartEndDate({
endDate: string;
}) {
let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff(
DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'),
DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss')
);
// this will make sure our start and end date's are correct

32
packages/db/src/services/conversion.service.ts
View File

@@ -31,14 +31,14 @@ export class ConversionService {
const funnelWindow = funnelOptions?.funnelWindow ?? 24;
const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';
const breakdownExpressions = breakdowns.map(
(b) => getSelectPropertyKey(b.name),
(b) => getSelectPropertyKey(b.name, projectId),
);
const breakdownSelects = breakdownExpressions.map(
(expr, index) => `${expr} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);
// Check if any breakdown uses profile fields and build profile JOIN if needed
// Check if any breakdown or filter uses profile fields
const profileBreakdowns = breakdowns.filter((b) =>
b.name.startsWith('profile.'),
);
@@ -71,6 +71,15 @@ export class ConversionService {
const events = onlyReportEvents(series);
// Check if any breakdown or filter uses group fields
const anyBreakdownOnGroup = breakdowns.some((b) =>
b.name.startsWith('group.'),
);
const anyFilterOnGroup = events.some((e) =>
e.filters?.some((f) => f.name.startsWith('group.')),
);
const needsGroupArrayJoin = anyBreakdownOnGroup || anyFilterOnGroup;
if (events.length !== 2) {
throw new Error('events must be an array of two events');
}
@@ -82,21 +91,25 @@ export class ConversionService {
const eventA = events[0]!;
const eventB = events[1]!;
const whereA = Object.values(
getEventFiltersWhereClause(eventA.filters),
getEventFiltersWhereClause(eventA.filters, projectId),
).join(' AND ');
const whereB = Object.values(
getEventFiltersWhereClause(eventB.filters),
getEventFiltersWhereClause(eventB.filters, projectId),
).join(' AND ');
const funnelWindowSeconds = funnelWindow * 3600;
// Build funnel conditions
const conditionA = whereA
? `(name = '${eventA.name}' AND ${whereA})`
: `name = '${eventA.name}'`;
? `(events.name = '${eventA.name}' AND ${whereA})`
: `events.name = '${eventA.name}'`;
const conditionB = whereB
? `(name = '${eventB.name}' AND ${whereB})`
: `name = '${eventB.name}'`;
? `(events.name = '${eventB.name}' AND ${whereB})`
: `events.name = '${eventB.name}'`;
const groupJoin = needsGroupArrayJoin
? `ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`
: '';
// Use windowFunnel approach - single scan, no JOIN
const query = clix(this.client, timezone)
@@ -126,8 +139,9 @@ export class ConversionService {
) as steps
FROM ${TABLE_NAMES.events}
${profileJoin}
${groupJoin}
WHERE project_id = '${projectId}'
AND name IN ('${eventA.name}', '${eventB.name}')
AND events.name IN ('${eventA.name}', '${eventB.name}')
AND created_at BETWEEN toDateTime('${startDate}') AND toDateTime('${endDate}')
GROUP BY ${group}${breakdownExpressions.length ? `, ${breakdownExpressions.join(', ')}` : ''})
`),

56
packages/db/src/services/event.service.ts
View File

@@ -32,14 +32,14 @@ export type IImportedEvent = Omit<
properties: Record<string, unknown>;
};
export type IServicePage = {
export interface IServicePage {
path: string;
count: number;
project_id: string;
first_seen: string;
title: string;
origin: string;
};
}
export interface IClickhouseBotEvent {
id: string;
@@ -92,6 +92,7 @@ export interface IClickhouseEvent {
sdk_name: string;
sdk_version: string;
revenue?: number;
groups: string[];
// They do not exist here. Just make ts happy for now
profile?: IServiceProfile;
@@ -143,6 +144,7 @@ export function transformSessionToEvent(
importedAt: undefined,
sdkName: undefined,
sdkVersion: undefined,
groups: [],
};
}
@@ -179,6 +181,7 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
sdkVersion: event.sdk_version,
profile: event.profile,
revenue: event.revenue,
groups: event.groups ?? [],
};
}
@@ -227,6 +230,7 @@ export interface IServiceEvent {
sdkName: string | undefined;
sdkVersion: string | undefined;
revenue?: number;
groups: string[];
}
type SelectHelper<T> = {
@@ -331,6 +335,7 @@ export async function getEvents(
projectId,
isExternal: false,
properties: {},
groups: [],
};
}
}
@@ -386,6 +391,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
sdk_name: payload.sdkName ?? '',
sdk_version: payload.sdkVersion ?? '',
revenue: payload.revenue,
groups: payload.groups ?? [],
};
const promises = [sessionBuffer.add(event), eventBuffer.add(event)];
@@ -434,6 +440,7 @@ export interface GetEventListOptions {
projectId: string;
profileId?: string;
sessionId?: string;
groupId?: string;
take: number;
cursor?: number | Date;
events?: string[] | null;
@@ -452,6 +459,7 @@ export async function getEventList(options: GetEventListOptions) {
projectId,
profileId,
sessionId,
groupId,
events,
filters,
startDate,
@@ -589,6 +597,10 @@ export async function getEventList(options: GetEventListOptions) {
sb.select.revenue = 'revenue';
}
if (select.groups) {
sb.select.groups = 'groups';
}
if (profileId) {
sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND device_id != '' AND profile_id = ${sqlstring.escape(profileId)} group by did) OR profile_id = ${sqlstring.escape(profileId)})`;
}
@@ -597,6 +609,10 @@ export async function getEventList(options: GetEventListOptions) {
sb.where.sessionId = `session_id = ${sqlstring.escape(sessionId)}`;
}
if (groupId) {
sb.where.groupId = `has(groups, ${sqlstring.escape(groupId)})`;
}
if (startDate && endDate) {
sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
}
@@ -611,7 +627,7 @@ export async function getEventList(options: GetEventListOptions) {
if (filters) {
sb.where = {
...sb.where,
...getEventFiltersWhereClause(filters),
...getEventFiltersWhereClause(filters, projectId),
};
// Join profiles table if any filter uses profile fields
@@ -622,6 +638,13 @@ export async function getEventList(options: GetEventListOptions) {
if (profileFilters.length > 0) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0])).join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
// Join groups table if any filter uses group fields
const groupFilters = filters.filter((f) => f.name.startsWith('group.'));
if (groupFilters.length > 0) {
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
}
}
sb.orderBy.created_at = 'created_at DESC, id ASC';
@@ -653,6 +676,7 @@ export async function getEventList(options: GetEventListOptions) {
export async function getEventsCount({
projectId,
profileId,
groupId,
events,
filters,
startDate,
@@ -664,6 +688,10 @@ export async function getEventsCount({
sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
}
if (groupId) {
sb.where.groupId = `has(groups, ${sqlstring.escape(groupId)})`;
}
if (startDate && endDate) {
sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
}
@@ -678,7 +706,7 @@ export async function getEventsCount({
if (filters) {
sb.where = {
...sb.where,
...getEventFiltersWhereClause(filters),
...getEventFiltersWhereClause(filters, projectId),
};
// Join profiles table if any filter uses profile fields
@@ -689,6 +717,13 @@ export async function getEventsCount({
if (profileFilters.length > 0) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0])).join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
// Join groups table if any filter uses group fields
const groupFilters = filters.filter((f) => f.name.startsWith('group.'));
if (groupFilters.length > 0) {
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
}
}
const res = await chQuery<{ count: number }>(
@@ -1052,8 +1087,19 @@ class EventService {
}
if (filters) {
q.rawWhere(
Object.values(getEventFiltersWhereClause(filters)).join(' AND ')
Object.values(
getEventFiltersWhereClause(filters, projectId)
).join(' AND ')
);
const groupFilters = filters.filter((f) =>
f.name.startsWith('group.')
);
if (groupFilters.length > 0) {
q.rawJoin('ARRAY JOIN groups AS _group_id');
q.rawJoin(
`LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`
);
}
}
},
session: (q) => {

33
packages/db/src/services/funnel.service.ts
View File

@@ -34,11 +34,11 @@ export class FunnelService {
return group === 'profile_id' ? 'profile_id' : 'session_id';
}
getFunnelConditions(events: IChartEvent[] = []): string[] {
getFunnelConditions(events: IChartEvent[] = [], projectId?: string): string[] {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.name = `events.name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
}
@@ -71,7 +71,7 @@ export class FunnelService {
additionalGroupBy?: string[];
group?: 'session_id' | 'profile_id';
}) {
const funnels = this.getFunnelConditions(eventSeries);
const funnels = this.getFunnelConditions(eventSeries, projectId);
const primaryKey = group === 'profile_id' ? 'profile_id' : 'session_id';
return clix(this.client, timezone)
@@ -90,7 +90,7 @@ export class FunnelService {
clix.datetime(endDate, 'toDateTime'),
])
.where(
'name',
'events.name',
'IN',
eventSeries.map((e) => e.name),
)
@@ -236,10 +236,18 @@ export class FunnelService {
const anyBreakdownOnProfile = breakdowns.some((b) =>
b.name.startsWith('profile.'),
);
const anyFilterOnGroup = eventSeries.some((e) =>
e.filters?.some((f) => f.name.startsWith('group.')),
);
const anyBreakdownOnGroup = breakdowns.some((b) =>
b.name.startsWith('group.'),
);
const needsGroupArrayJoin =
anyFilterOnGroup || anyBreakdownOnGroup || funnelGroup === 'group';
// Create the funnel CTE (session-level)
const breakdownSelects = breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
(b, index) => `${getSelectPropertyKey(b.name, projectId)} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
@@ -277,8 +285,21 @@ export class FunnelService {
);
}
if (needsGroupArrayJoin) {
funnelCte.rawJoin('ARRAY JOIN groups AS _group_id');
funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');
}
// Base funnel query with CTEs
const funnelQuery = clix(this.client, timezone);
if (needsGroupArrayJoin) {
funnelQuery.with(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`,
);
}
funnelQuery.with('session_funnel', funnelCte);
// windowFunnel is computed per the primary key (profile_id or session_id),

363
packages/db/src/services/group.service.ts Normal file
View File

@@ -0,0 +1,363 @@
import { toDots } from '@openpanel/common';
import sqlstring from 'sqlstring';
import {
ch,
chQuery,
formatClickhouseDate,
TABLE_NAMES,
} from '../clickhouse/client';
import type { IServiceProfile } from './profile.service';
import { getProfiles } from './profile.service';
export type IServiceGroup = {
id: string;
projectId: string;
type: string;
name: string;
properties: Record<string, unknown>;
createdAt: Date;
updatedAt: Date;
};
export type IServiceUpsertGroup = {
id: string;
projectId: string;
type: string;
name: string;
properties?: Record<string, unknown>;
};
type IClickhouseGroup = {
project_id: string;
id: string;
type: string;
name: string;
properties: Record<string, string>;
created_at: string;
version: string;
};
function transformGroup(row: IClickhouseGroup): IServiceGroup {
return {
id: row.id,
projectId: row.project_id,
type: row.type,
name: row.name,
properties: row.properties,
createdAt: new Date(row.created_at),
updatedAt: new Date(Number(row.version)),
};
}
async function writeGroupToCh(
group: {
id: string;
projectId: string;
type: string;
name: string;
properties: Record<string, string>;
createdAt?: Date;
},
deleted = 0
) {
await ch.insert({
format: 'JSONEachRow',
table: TABLE_NAMES.groups,
values: [
{
project_id: group.projectId,
id: group.id,
type: group.type,
name: group.name,
properties: group.properties,
created_at: formatClickhouseDate(group.createdAt ?? new Date()),
version: Date.now(),
deleted,
},
],
});
}
export async function upsertGroup(input: IServiceUpsertGroup) {
const existing = await getGroupById(input.id, input.projectId);
await writeGroupToCh({
id: input.id,
projectId: input.projectId,
type: input.type,
name: input.name,
properties: toDots({
...(existing?.properties ?? {}),
...(input.properties ?? {}),
}),
createdAt: existing?.createdAt,
});
}
export async function getGroupById(
id: string,
projectId: string
): Promise<IServiceGroup | null> {
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id = ${sqlstring.escape(id)}
AND deleted = 0
`);
return rows[0] ? transformGroup(rows[0]) : null;
}
export async function getGroupList({
projectId,
cursor,
take,
search,
type,
}: {
projectId: string;
cursor?: number;
take: number;
search?: string;
type?: string;
}): Promise<IServiceGroup[]> {
const conditions = [
`project_id = ${sqlstring.escape(projectId)}`,
'deleted = 0',
...(type ? [`type = ${sqlstring.escape(type)}`] : []),
...(search
? [
`(name ILIKE ${sqlstring.escape(`%${search}%`)} OR id ILIKE ${sqlstring.escape(`%${search}%`)})`,
]
: []),
];
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE ${conditions.join(' AND ')}
ORDER BY created_at DESC
LIMIT ${take}
OFFSET ${Math.max(0, (cursor ?? 0) * take)}
`);
return rows.map(transformGroup);
}
export async function getGroupListCount({
projectId,
type,
search,
}: {
projectId: string;
type?: string;
search?: string;
}): Promise<number> {
const conditions = [
`project_id = ${sqlstring.escape(projectId)}`,
'deleted = 0',
...(type ? [`type = ${sqlstring.escape(type)}`] : []),
...(search
? [
`(name ILIKE ${sqlstring.escape(`%${search}%`)} OR id ILIKE ${sqlstring.escape(`%${search}%`)})`,
]
: []),
];
const rows = await chQuery<{ count: number }>(`
SELECT count() as count
FROM ${TABLE_NAMES.groups} FINAL
WHERE ${conditions.join(' AND ')}
`);
return rows[0]?.count ?? 0;
}
export async function getGroupTypes(projectId: string): Promise<string[]> {
const rows = await chQuery<{ type: string }>(`
SELECT DISTINCT type
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND deleted = 0
`);
return rows.map((r) => r.type);
}
export async function createGroup(input: IServiceUpsertGroup) {
await upsertGroup(input);
return getGroupById(input.id, input.projectId);
}
export async function updateGroup(
id: string,
projectId: string,
data: { type?: string; name?: string; properties?: Record<string, unknown> }
) {
const existing = await getGroupById(id, projectId);
if (!existing) {
throw new Error(`Group ${id} not found`);
}
const mergedProperties = {
...(existing.properties ?? {}),
...(data.properties ?? {}),
};
const normalizedProperties = toDots(
mergedProperties as Record<string, unknown>
);
const updated = {
id,
projectId,
type: data.type ?? existing.type,
name: data.name ?? existing.name,
properties: normalizedProperties,
createdAt: existing.createdAt,
};
await writeGroupToCh(updated);
return { ...existing, ...updated };
}
export async function deleteGroup(id: string, projectId: string) {
const existing = await getGroupById(id, projectId);
if (!existing) {
throw new Error(`Group ${id} not found`);
}
await writeGroupToCh(
{
id,
projectId,
type: existing.type,
name: existing.name,
properties: existing.properties as Record<string, string>,
createdAt: existing.createdAt,
},
1
);
return existing;
}
export async function getGroupPropertyKeys(
projectId: string
): Promise<string[]> {
const rows = await chQuery<{ key: string }>(`
SELECT DISTINCT arrayJoin(mapKeys(properties)) as key
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND deleted = 0
`);
return rows.map((r) => r.key).sort();
}
export type IServiceGroupStats = {
groupId: string;
memberCount: number;
lastActiveAt: Date | null;
};
export async function getGroupStats(
projectId: string,
groupIds: string[]
): Promise<Map<string, IServiceGroupStats>> {
if (groupIds.length === 0) {
return new Map();
}
const rows = await chQuery<{
group_id: string;
member_count: number;
last_active_at: string;
}>(`
SELECT
g AS group_id,
uniqExact(profile_id) AS member_count,
max(created_at) AS last_active_at
FROM ${TABLE_NAMES.events}
ARRAY JOIN groups AS g
WHERE project_id = ${sqlstring.escape(projectId)}
AND g IN (${groupIds.map((id) => sqlstring.escape(id)).join(',')})
AND profile_id != device_id
GROUP BY g
`);
return new Map(
rows.map((r) => [
r.group_id,
{
groupId: r.group_id,
memberCount: r.member_count,
lastActiveAt: r.last_active_at ? new Date(r.last_active_at) : null,
},
])
);
}
export async function getGroupsByIds(
projectId: string,
ids: string[]
): Promise<IServiceGroup[]> {
if (ids.length === 0) {
return [];
}
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id IN (${ids.map((id) => sqlstring.escape(id)).join(',')})
AND deleted = 0
`);
return rows.map(transformGroup);
}
export async function getGroupMemberProfiles({
projectId,
groupId,
cursor,
take,
search,
}: {
projectId: string;
groupId: string;
cursor?: number;
take: number;
search?: string;
}): Promise<{ data: IServiceProfile[]; count: number }> {
const offset = Math.max(0, (cursor ?? 0) * take);
const searchCondition = search?.trim()
? `AND (email ILIKE ${sqlstring.escape(`%${search.trim()}%`)} OR first_name ILIKE ${sqlstring.escape(`%${search.trim()}%`)} OR last_name ILIKE ${sqlstring.escape(`%${search.trim()}%`)})`
: '';
// count() OVER () is evaluated after JOINs/WHERE but before LIMIT,
// so we get the total match count and the paginated IDs in one query.
const rows = await chQuery<{ profile_id: string; total_count: number }>(`
SELECT
gm.profile_id,
count() OVER () AS total_count
FROM (
SELECT profile_id, max(created_at) AS last_seen
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(groupId)})
AND profile_id != device_id
GROUP BY profile_id
) gm
INNER JOIN (
SELECT id FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
${searchCondition}
) p ON p.id = gm.profile_id
ORDER BY gm.last_seen DESC
LIMIT ${take}
OFFSET ${offset}
`);
const count = rows[0]?.total_count ?? 0;
const profileIds = rows.map((r) => r.profile_id);
if (profileIds.length === 0) {
return { data: [], count };
}
const profiles = await getProfiles(profileIds, projectId);
const byId = new Map(profiles.map((p) => [p.id, p]));
const data = profileIds
.map((id) => byId.get(id))
.filter(Boolean) as IServiceProfile[];
return { data, count };
}

2
packages/db/src/services/import.service.ts
View File

@@ -355,6 +355,7 @@ export async function createSessionsStartEndEvents(
profile_id: session.profile_id,
project_id: session.project_id,
session_id: session.session_id,
groups: [],
path: firstPath,
origin: firstOrigin,
referrer: firstReferrer,
@@ -390,6 +391,7 @@ export async function createSessionsStartEndEvents(
profile_id: session.profile_id,
project_id: session.project_id,
session_id: session.session_id,
groups: [],
path: lastPath,
origin: lastOrigin,
referrer: firstReferrer,

9
packages/db/src/services/profile.service.ts
View File

@@ -165,7 +165,8 @@ export async function getProfiles(ids: string[], projectId: string) {
any(nullIf(avatar, '')) as avatar,
last_value(is_external) as is_external,
any(properties) as properties,
any(created_at) as created_at
any(created_at) as created_at,
any(groups) as groups
FROM ${TABLE_NAMES.profiles}
WHERE
project_id = ${sqlstring.escape(projectId)} AND
@@ -232,6 +233,7 @@ export interface IServiceProfile {
createdAt: Date;
isExternal: boolean;
projectId: string;
groups: string[];
properties: Record<string, unknown> & {
region?: string;
country?: string;
@@ -259,6 +261,7 @@ export interface IClickhouseProfile {
project_id: string;
is_external: boolean;
created_at: string;
groups: string[];
}
export interface IServiceUpsertProfile {
@@ -270,6 +273,7 @@ export interface IServiceUpsertProfile {
avatar?: string;
properties?: Record<string, unknown>;
isExternal: boolean;
groups?: string[];
}
export function transformProfile({
@@ -288,6 +292,7 @@ export function transformProfile({
id: profile.id,
email: profile.email,
avatar: profile.avatar,
groups: profile.groups ?? [],
};
}
@@ -301,6 +306,7 @@ export function upsertProfile(
properties,
projectId,
isExternal,
groups,
}: IServiceUpsertProfile,
isFromEvent = false
) {
@@ -314,6 +320,7 @@ export function upsertProfile(
project_id: projectId,
created_at: formatClickhouseDate(new Date()),
is_external: isExternal,
groups: groups ?? [],
};
return profileBuffer.add(profile, isFromEvent);

5
packages/db/src/services/session.service.ts
View File

@@ -55,6 +55,7 @@ export interface IClickhouseSession {
version: number;
// Dynamically added
has_replay?: boolean;
groups: string[];
}
export interface IServiceSession {
@@ -95,6 +96,7 @@ export interface IServiceSession {
revenue: number;
profile?: IServiceProfile;
hasReplay?: boolean;
groups: string[];
}
export interface GetSessionListOptions {
@@ -152,6 +154,7 @@ export function transformSession(session: IClickhouseSession): IServiceSession {
revenue: session.revenue,
profile: undefined,
hasReplay: session.has_replay,
groups: session.groups,
};
}
@@ -244,6 +247,7 @@ export async function getSessionList(options: GetSessionListOptions) {
'screen_view_count',
'event_count',
'revenue',
'groups',
];
columns.forEach((column) => {
@@ -292,6 +296,7 @@ export async function getSessionList(options: GetSessionListOptions) {
projectId,
isExternal: false,
properties: {},
groups: [],
},
}));