zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: group analytics

Browse Source 
* wip

* wip

* wip

* wip

* wip

* add buffer

* wip

* wip

* fixes

* fix

* wip

* group validation

* fix group issues

* docs: add groups
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-20 10:46:09 +01:00
committed by GitHub
parent 88a2d876ce
commit 11e9ecac1a
99 changed files with 5944 additions and 1432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
packages/common/scripts/get-referrers.ts
View File

@@ -1,6 +1,5 @@
import fs from 'node:fs';
import path from 'node:path';
import { dirname } from 'node:path';
import path, { dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
@@ -44,6 +43,65 @@ const extraReferrers = {
'squarespace.com': { type: 'commerce', name: 'Squarespace' },
'stackoverflow.com': { type: 'tech', name: 'Stack Overflow' },
'teams.microsoft.com': { type: 'social', name: 'Microsoft Teams' },
'chat.com': { type: 'ai', name: 'Chat.com' },
'chatgpt.com': { type: 'ai', name: 'ChatGPT' },
'openai.com': { type: 'ai', name: 'OpenAI' },
'anthropic.com': { type: 'ai', name: 'Anthropic' },
'claude.ai': { type: 'ai', name: 'Claude' },
'gemini.google.com': { type: 'ai', name: 'Google Gemini' },
'bard.google.com': { type: 'ai', name: 'Google Bard' },
'copilot.microsoft.com': { type: 'ai', name: 'Microsoft Copilot' },
'copilot.cloud.microsoft': { type: 'ai', name: 'Microsoft Copilot' },
'perplexity.ai': { type: 'ai', name: 'Perplexity' },
'you.com': { type: 'ai', name: 'You.com' },
'poe.com': { type: 'ai', name: 'Poe' },
'phind.com': { type: 'ai', name: 'Phind' },
'huggingface.co': { type: 'ai', name: 'Hugging Face' },
'hf.co': { type: 'ai', name: 'Hugging Face' },
'character.ai': { type: 'ai', name: 'Character.AI' },
'meta.ai': { type: 'ai', name: 'Meta AI' },
'mistral.ai': { type: 'ai', name: 'Mistral' },
'chat.mistral.ai': { type: 'ai', name: 'Mistral Le Chat' },
'deepseek.com': { type: 'ai', name: 'DeepSeek' },
'chat.deepseek.com': { type: 'ai', name: 'DeepSeek Chat' },
'pi.ai': { type: 'ai', name: 'Pi' },
'inflection.ai': { type: 'ai', name: 'Inflection' },
'cohere.com': { type: 'ai', name: 'Cohere' },
'coral.cohere.com': { type: 'ai', name: 'Cohere Coral' },
'jasper.ai': { type: 'ai', name: 'Jasper' },
'writesonic.com': { type: 'ai', name: 'Writesonic' },
'copy.ai': { type: 'ai', name: 'Copy.ai' },
'rytr.me': { type: 'ai', name: 'Rytr' },
'notion.ai': { type: 'ai', name: 'Notion AI' },
'grammarly.com': { type: 'ai', name: 'Grammarly' },
'grok.com': { type: 'ai', name: 'Grok' },
'x.ai': { type: 'ai', name: 'xAI' },
'aistudio.google.com': { type: 'ai', name: 'Google AI Studio' },
'labs.google.com': { type: 'ai', name: 'Google Labs' },
'ai.google': { type: 'ai', name: 'Google AI' },
'forefront.ai': { type: 'ai', name: 'Forefront' },
'together.ai': { type: 'ai', name: 'Together AI' },
'groq.com': { type: 'ai', name: 'Groq' },
'replicate.com': { type: 'ai', name: 'Replicate' },
'vercel.ai': { type: 'ai', name: 'Vercel AI' },
'v0.dev': { type: 'ai', name: 'v0' },
'bolt.new': { type: 'ai', name: 'Bolt' },
'replit.com': { type: 'ai', name: 'Replit' },
'cursor.com': { type: 'ai', name: 'Cursor' },
'tabnine.com': { type: 'ai', name: 'Tabnine' },
'codeium.com': { type: 'ai', name: 'Codeium' },
'sourcegraph.com': { type: 'ai', name: 'Sourcegraph Cody' },
'kimi.moonshot.cn': { type: 'ai', name: 'Kimi' },
'moonshot.ai': { type: 'ai', name: 'Moonshot AI' },
'doubao.com': { type: 'ai', name: 'Doubao' },
'tongyi.aliyun.com': { type: 'ai', name: 'Tongyi Qianwen' },
'yiyan.baidu.com': { type: 'ai', name: 'Ernie Bot' },
'chatglm.cn': { type: 'ai', name: 'ChatGLM' },
'zhipu.ai': { type: 'ai', name: 'Zhipu AI' },
'minimax.chat': { type: 'ai', name: 'MiniMax' },
'lmsys.org': { type: 'ai', name: 'LMSYS' },
'chat.lmsys.org': { type: 'ai', name: 'LMSYS Chat' },
'llama.meta.com': { type: 'ai', name: 'Meta Llama' },
};
function transform(data: any) {
@@ -67,7 +125,7 @@ async function main() {
// Get document, or throw exception on error
try {
const data = await fetch(
'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json',
'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json'
).then((res) => res.json());
fs.writeFileSync(
@@ -82,11 +140,11 @@ async function main() {
{
...transform(data),
...extraReferrers,
},
}
)} as const;`,
'export default referrers;',
].join('\n'),
'utf-8',
'utf-8'
);
} catch (e) {
console.log(e);

14
packages/common/src/math.ts
View File

@@ -20,11 +20,17 @@ export const average = (arr: (number | null)[], includeZero = false) => {
export const sum = (arr: (number | null | undefined)[]): number =>
round(arr.filter(isNumber).reduce((acc, item) => acc + item, 0));
export const min = (arr: (number | null | undefined)[]): number =>
Math.min(...arr.filter(isNumber));
export const min = (arr: (number | null | undefined)[]): number => {
const filtered = arr.filter(isNumber);
if (filtered.length === 0) return 0;
return filtered.reduce((a, b) => (b < a ? b : a), filtered[0]!);
};
export const max = (arr: (number | null | undefined)[]): number =>
Math.max(...arr.filter(isNumber));
export const max = (arr: (number | null | undefined)[]): number => {
const filtered = arr.filter(isNumber);
if (filtered.length === 0) return 0;
return filtered.reduce((a, b) => (b > a ? b : a), filtered[0]!);
};
export const isFloat = (n: number) => n % 1 !== 0;

7
packages/constants/index.ts
View File

@@ -113,6 +113,7 @@ export const chartSegments = {
event: 'All events',
user: 'Unique users',
session: 'Unique sessions',
group: 'Unique groups',
user_average: 'Average users',
one_event_per_user: 'One event per user',
property_sum: 'Sum of property',
@@ -195,7 +196,7 @@ export const metrics = {
} as const;
export function isMinuteIntervalEnabledByRange(
range: keyof typeof timeWindows,
range: keyof typeof timeWindows
) {
return range === '30min' || range === 'lastHour';
}
@@ -210,7 +211,7 @@ export function isHourIntervalEnabledByRange(range: keyof typeof timeWindows) {
}
export function getDefaultIntervalByRange(
range: keyof typeof timeWindows,
range: keyof typeof timeWindows
): keyof typeof intervals {
if (range === '30min' || range === 'lastHour') {
return 'minute';
@@ -231,7 +232,7 @@ export function getDefaultIntervalByRange(
export function getDefaultIntervalByDates(
startDate: string | null,
endDate: string | null,
endDate: string | null
): null | keyof typeof intervals {
if (startDate && endDate) {
if (isSameDay(startDate, endDate)) {

66
packages/db/code-migrations/11-add-groups.ts Normal file
View File

@@ -0,0 +1,66 @@
import fs from 'node:fs';
import path from 'node:path';
import { TABLE_NAMES } from '../src/clickhouse/client';
import {
addColumns,
createTable,
runClickhouseMigrationCommands,
} from '../src/clickhouse/migration';
import { getIsCluster } from './helpers';
export async function up() {
const isClustered = getIsCluster();
const sqls: string[] = [
...addColumns(
'events',
['`groups` Array(String) DEFAULT [] CODEC(ZSTD(3)) AFTER session_id'],
isClustered
),
...addColumns(
'sessions',
['`groups` Array(String) DEFAULT [] CODEC(ZSTD(3)) AFTER device_id'],
isClustered
),
...addColumns(
'profiles',
['`groups` Array(String) DEFAULT [] CODEC(ZSTD(3)) AFTER project_id'],
isClustered
),
...createTable({
name: TABLE_NAMES.groups,
columns: [
'`id` String',
'`project_id` String',
'`type` String',
'`name` String',
'`properties` Map(String, String)',
'`created_at` DateTime',
'`version` UInt64',
'`deleted` UInt8 DEFAULT 0',
],
engine: 'ReplacingMergeTree(version, deleted)',
orderBy: ['project_id', 'id'],
distributionHash: 'cityHash64(project_id, id)',
replicatedVersion: '1',
isClustered,
}),
];
fs.writeFileSync(
path.join(import.meta.filename.replace('.ts', '.sql')),
sqls
.map((sql) =>
sql
.trim()
.replace(/;$/, '')
.replace(/\n{2,}/g, '\n')
.concat(';')
)
.join('\n\n---\n\n')
);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}

60
packages/db/index.ts
View File

@@ -1,35 +1,37 @@
export * from './src/prisma-client';
export * from './src/clickhouse/client';
export * from './src/sql-builder';
export * from './src/services/chart.service';
export * from './src/engine';
export * from './src/services/clients.service';
export * from './src/services/dashboard.service';
export * from './src/services/event.service';
export * from './src/services/organization.service';
export * from './src/services/profile.service';
export * from './src/services/project.service';
export * from './src/services/reports.service';
export * from './src/services/salt.service';
export * from './src/services/share.service';
export * from './src/services/session.service';
export * from './src/services/funnel.service';
export * from './src/services/conversion.service';
export * from './src/services/sankey.service';
export * from './src/services/user.service';
export * from './src/services/reference.service';
export * from './src/services/id.service';
export * from './src/services/retention.service';
export * from './src/services/notification.service';
export * from './src/services/access.service';
export * from './src/services/delete.service';
export * from './src/buffers';
export * from './src/types';
export * from './src/clickhouse/client';
export * from './src/clickhouse/query-builder';
export * from './src/encryption';
export * from './src/engine';
export * from './src/engine';
export * from './src/gsc';
export * from './src/prisma-client';
export * from './src/services/access.service';
export * from './src/services/chart.service';
export * from './src/services/clients.service';
export * from './src/services/conversion.service';
export * from './src/services/dashboard.service';
export * from './src/services/delete.service';
export * from './src/services/event.service';
export * from './src/services/funnel.service';
export * from './src/services/group.service';
export * from './src/services/id.service';
export * from './src/services/import.service';
export * from './src/services/insights';
export * from './src/services/notification.service';
export * from './src/services/organization.service';
export * from './src/services/overview.service';
export * from './src/services/pages.service';
export * from './src/services/insights';
export * from './src/services/profile.service';
export * from './src/services/project.service';
export * from './src/services/reference.service';
export * from './src/services/reports.service';
export * from './src/services/retention.service';
export * from './src/services/salt.service';
export * from './src/services/sankey.service';
export * from './src/services/session.service';
export * from './src/services/share.service';
export * from './src/services/user.service';
export * from './src/session-context';
export * from './src/gsc';
export * from './src/encryption';
export * from './src/sql-builder';
export * from './src/types';

2
packages/db/prisma/schema.prisma
View File

@@ -199,7 +199,6 @@ model Project {
meta EventMeta[]
references Reference[]
access ProjectAccess[]
notificationRules NotificationRule[]
notifications Notification[]
imports Import[]
@@ -215,6 +214,7 @@ model Project {
@@map("projects")
}
enum AccessLevel {
read
write

23
packages/db/src/buffers/event-buffer.ts
View File

@@ -1,11 +1,7 @@
import { getSafeJson } from '@openpanel/json';
import {
type Redis,
getRedisCache,
publishEvent,
} from '@openpanel/redis';
import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import { type IClickhouseEvent } from '../services/event.service';
import type { IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
export class EventBuffer extends BaseBuffer {
@@ -95,7 +91,7 @@ export class EventBuffer extends BaseBuffer {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id,
event.profile_id
);
}
}
@@ -116,7 +112,7 @@ export class EventBuffer extends BaseBuffer {
error,
eventCount: eventsToFlush.length,
flushRetryCount: this.flushRetryCount,
},
}
);
} finally {
this.isFlushing = false;
@@ -137,7 +133,7 @@ export class EventBuffer extends BaseBuffer {
const queueEvents = await redis.lrange(
this.queueKey,
0,
this.batchSize - 1,
this.batchSize - 1
);
if (queueEvents.length === 0) {
@@ -149,6 +145,9 @@ export class EventBuffer extends BaseBuffer {
for (const eventStr of queueEvents) {
const event = getSafeJson<IClickhouseEvent>(eventStr);
if (event) {
if (!Array.isArray(event.groups)) {
event.groups = [];
}
eventsToClickhouse.push(event);
}
}
@@ -161,7 +160,7 @@ export class EventBuffer extends BaseBuffer {
eventsToClickhouse.sort(
(a, b) =>
new Date(a.created_at || 0).getTime() -
new Date(b.created_at || 0).getTime(),
new Date(b.created_at || 0).getTime()
);
this.logger.info('Inserting events into ClickHouse', {
@@ -181,7 +180,7 @@ export class EventBuffer extends BaseBuffer {
for (const event of eventsToClickhouse) {
countByProject.set(
event.project_id,
(countByProject.get(event.project_id) ?? 0) + 1,
(countByProject.get(event.project_id) ?? 0) + 1
);
}
for (const [projectId, count] of countByProject) {
@@ -222,7 +221,7 @@ export class EventBuffer extends BaseBuffer {
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string,
profileId: string
) {
const key = `${projectId}:${profileId}`;
const now = Date.now();

195
packages/db/src/buffers/group-buffer.ts Normal file
View File

@@ -0,0 +1,195 @@
import { toDots } from '@openpanel/common';
import { getSafeJson } from '@openpanel/json';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import sqlstring from 'sqlstring';
import { ch, chQuery, formatClickhouseDate, TABLE_NAMES } from '../clickhouse/client';
import { BaseBuffer } from './base-buffer';
type IGroupBufferEntry = {
project_id: string;
id: string;
type: string;
name: string;
properties: Record<string, string>;
created_at: string;
version: string;
deleted: number;
};
type IGroupCacheEntry = {
id: string;
project_id: string;
type: string;
name: string;
properties: Record<string, string>;
created_at: string;
};
export type IGroupBufferInput = {
id: string;
projectId: string;
type: string;
name: string;
properties?: Record<string, unknown>;
};
export class GroupBuffer extends BaseBuffer {
private batchSize = process.env.GROUP_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.GROUP_BUFFER_BATCH_SIZE, 10)
: 200;
private chunkSize = process.env.GROUP_BUFFER_CHUNK_SIZE
? Number.parseInt(process.env.GROUP_BUFFER_CHUNK_SIZE, 10)
: 1000;
private ttlInSeconds = process.env.GROUP_BUFFER_TTL_IN_SECONDS
? Number.parseInt(process.env.GROUP_BUFFER_TTL_IN_SECONDS, 10)
: 60 * 60;
private readonly redisKey = 'group-buffer';
private readonly redisCachePrefix = 'group-cache:';
private redis: Redis;
constructor() {
super({
name: 'group',
onFlush: async () => {
await this.processBuffer();
},
});
this.redis = getRedisCache();
}
private getCacheKey(projectId: string, id: string) {
return `${this.redisCachePrefix}${projectId}:${id}`;
}
private async fetchFromCache(
projectId: string,
id: string
): Promise<IGroupCacheEntry | null> {
const raw = await this.redis.get(this.getCacheKey(projectId, id));
if (!raw) return null;
return getSafeJson<IGroupCacheEntry>(raw);
}
private async fetchFromClickhouse(
projectId: string,
id: string
): Promise<IGroupCacheEntry | null> {
const rows = await chQuery<IGroupCacheEntry>(`
SELECT project_id, id, type, name, properties, created_at
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id = ${sqlstring.escape(id)}
AND deleted = 0
`);
return rows[0] ?? null;
}
async add(input: IGroupBufferInput): Promise<void> {
try {
const cacheKey = this.getCacheKey(input.projectId, input.id);
const existing =
(await this.fetchFromCache(input.projectId, input.id)) ??
(await this.fetchFromClickhouse(input.projectId, input.id));
const mergedProperties = toDots({
...(existing?.properties ?? {}),
...(input.properties ?? {}),
}) as Record<string, string>;
const entry: IGroupBufferEntry = {
project_id: input.projectId,
id: input.id,
type: input.type,
name: input.name,
properties: mergedProperties,
created_at: formatClickhouseDate(
existing?.created_at ? new Date(existing.created_at) : new Date()
),
version: String(Date.now()),
deleted: 0,
};
if (
existing &&
existing.type === entry.type &&
existing.name === entry.name &&
shallowEqual(existing.properties, entry.properties)
) {
this.logger.debug('Group not changed, skipping', { id: input.id });
return;
}
const cacheEntry: IGroupCacheEntry = {
id: entry.id,
project_id: entry.project_id,
type: entry.type,
name: entry.name,
properties: entry.properties,
created_at: entry.created_at,
};
const result = await this.redis
.multi()
.set(cacheKey, JSON.stringify(cacheEntry), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(entry))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add group to Redis', { input });
return;
}
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
} catch (error) {
this.logger.error('Failed to add group', { error, input });
}
}
async processBuffer(): Promise<void> {
try {
this.logger.debug('Starting group buffer processing');
const items = await this.redis.lrange(this.redisKey, 0, this.batchSize - 1);
if (items.length === 0) {
this.logger.debug('No groups to process');
return;
}
this.logger.debug(`Processing ${items.length} groups in buffer`);
const parsed = items.map((i) => getSafeJson<IGroupBufferEntry>(i));
for (const chunk of this.chunks(parsed, this.chunkSize)) {
await ch.insert({
table: TABLE_NAMES.groups,
values: chunk,
format: 'JSONEachRow',
});
}
await this.redis
.multi()
.ltrim(this.redisKey, items.length, -1)
.decrby(this.bufferCounterKey, items.length)
.exec();
this.logger.debug('Successfully completed group processing', {
totalGroups: items.length,
});
} catch (error) {
this.logger.error('Failed to process buffer', { error });
}
}
async getBufferSize(): Promise<number> {
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
}
}

2
packages/db/src/buffers/index.ts
View File

@@ -1,5 +1,6 @@
import { BotBuffer as BotBufferRedis } from './bot-buffer';
import { EventBuffer as EventBufferRedis } from './event-buffer';
import { GroupBuffer } from './group-buffer';
import { ProfileBackfillBuffer } from './profile-backfill-buffer';
import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
import { ReplayBuffer } from './replay-buffer';
@@ -11,6 +12,7 @@ export const botBuffer = new BotBufferRedis();
export const sessionBuffer = new SessionBuffer();
export const profileBackfillBuffer = new ProfileBackfillBuffer();
export const replayBuffer = new ReplayBuffer();
export const groupBuffer = new GroupBuffer();
export type { ProfileBackfillEntry } from './profile-backfill-buffer';
export type { IClickhouseSessionReplayChunk } from './replay-buffer';

151
packages/db/src/buffers/profile-buffer.test.ts Normal file
View File

@@ -0,0 +1,151 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { getSafeJson } from '@openpanel/json';
import type { IClickhouseProfile } from '../services/profile.service';
// Mock chQuery to avoid hitting real ClickHouse
vi.mock('../clickhouse/client', () => ({
ch: {
insert: vi.fn().mockResolvedValue(undefined),
},
chQuery: vi.fn().mockResolvedValue([]),
TABLE_NAMES: {
profiles: 'profiles',
},
}));
import { ProfileBuffer } from './profile-buffer';
import { chQuery } from '../clickhouse/client';
const redis = getRedisCache();
function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile {
return {
id: 'profile-1',
project_id: 'project-1',
first_name: '',
last_name: '',
email: '',
avatar: '',
properties: {},
is_external: true,
created_at: new Date().toISOString(),
groups: [],
...overrides,
};
}
beforeEach(async () => {
await redis.flushdb();
vi.mocked(chQuery).mockResolvedValue([]);
});
afterAll(async () => {
try {
await redis.quit();
} catch {}
});
describe('ProfileBuffer', () => {
let profileBuffer: ProfileBuffer;
beforeEach(() => {
profileBuffer = new ProfileBuffer();
});
it('adds a profile to the buffer', async () => {
const profile = makeProfile({ first_name: 'John', email: 'john@example.com' });
const sizeBefore = await profileBuffer.getBufferSize();
await profileBuffer.add(profile);
const sizeAfter = await profileBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore + 1);
});
it('merges subsequent updates via cache (sequential calls)', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Sequential: identify first, then group
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
// Second add should read the cached identify profile and merge groups in
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent identify + group calls preserve all data', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Both calls run concurrently — the per-profile lock serializes them so the
// second one reads the first's result from cache and merges correctly.
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent writes produce one merged buffer entry', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
const sizeBefore = await profileBuffer.getBufferSize();
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const sizeAfter = await profileBuffer.getBufferSize();
// The second add merges into the first — only 2 buffer entries total
// (one from identify, one merged update with group)
expect(sizeAfter).toBe(sizeBefore + 2);
// The last entry in the buffer should have both name and group
const rawEntries = await redis.lrange('profile-buffer', 0, -1);
const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e));
const lastEntry = entries[entries.length - 1];
expect(lastEntry?.first_name).toBe('John');
expect(lastEntry?.groups).toContain('group-abc');
});
});

200
packages/db/src/buffers/profile-buffer.ts
View File

@@ -1,9 +1,10 @@
import { deepMergeObjects } from '@openpanel/common';
import { generateSecureId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json';
import type { ILogger } from '@openpanel/logger';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import { omit } from 'ramda';
import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
import type { IClickhouseProfile } from '../services/profile.service';
@@ -24,6 +25,15 @@ export class ProfileBuffer extends BaseBuffer {
private readonly redisProfilePrefix = 'profile-cache:';
private redis: Redis;
private releaseLockSha: string | null = null;
private readonly releaseLockScript = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
else
return 0
end
`;
constructor() {
super({
@@ -33,6 +43,9 @@ export class ProfileBuffer extends BaseBuffer {
},
});
this.redis = getRedisCache();
this.redis.script('LOAD', this.releaseLockScript).then((sha) => {
this.releaseLockSha = sha as string;
});
}
private getProfileCacheKey({
@@ -45,6 +58,42 @@ export class ProfileBuffer extends BaseBuffer {
return `${this.redisProfilePrefix}${projectId}:${profileId}`;
}
private async withProfileLock<T>(
profileId: string,
projectId: string,
fn: () => Promise<T>
): Promise<T> {
const lockKey = `profile-lock:${projectId}:${profileId}`;
const lockId = generateSecureId('lock');
const maxRetries = 20;
const retryDelayMs = 50;
for (let i = 0; i < maxRetries; i++) {
const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');
if (acquired === 'OK') {
try {
return await fn();
} finally {
if (this.releaseLockSha) {
await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);
} else {
await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);
}
}
}
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
}
this.logger.error(
'Failed to acquire profile lock, proceeding without lock',
{
profileId,
projectId,
}
);
return fn();
}
async alreadyExists(profile: IClickhouseProfile) {
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
@@ -67,83 +116,94 @@ export class ProfileBuffer extends BaseBuffer {
return;
}
const existingProfile = await this.fetchProfile(profile, logger);
await this.withProfileLock(profile.id, profile.project_id, async () => {
const existingProfile = await this.fetchProfile(profile, logger);
// Delete any properties that are not server related if we have a non-server profile
if (
existingProfile?.properties.device !== 'server' &&
profile.properties.device === 'server'
) {
profile.properties = omit(
[
'city',
'country',
'region',
'longitude',
'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
profile.properties
);
}
// Delete any properties that are not server related if we have a non-server profile
if (
existingProfile?.properties.device !== 'server' &&
profile.properties.device === 'server'
) {
profile.properties = omit(
[
'city',
'country',
'region',
'longitude',
'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
profile.properties
);
}
const mergedProfile: IClickhouseProfile = existingProfile
? deepMergeObjects(existingProfile, omit(['created_at'], profile))
: profile;
const mergedProfile: IClickhouseProfile = existingProfile
? {
...deepMergeObjects(
existingProfile,
omit(['created_at', 'groups'], profile)
),
groups: uniq([
...(existingProfile.groups ?? []),
...(profile.groups ?? []),
]),
}
: profile;
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
this.logger.debug('Merged profile will be inserted', {
mergedProfile,
existingProfile,
profile,
});
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
const result = await this.redis
.multi()
.set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(mergedProfile))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add profile to Redis', {
this.logger.debug('Merged profile will be inserted', {
mergedProfile,
existingProfile,
profile,
cacheKey,
});
return;
}
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
this.logger.debug('Current buffer length', {
bufferLength,
batchSize: this.batchSize,
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
const result = await this.redis
.multi()
.set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(mergedProfile))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add profile to Redis', {
profile,
cacheKey,
});
return;
}
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
this.logger.debug('Current buffer length', {
bufferLength,
batchSize: this.batchSize,
});
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
});
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
} catch (error) {
this.logger.error('Failed to add profile', { error, profile });
}

7
packages/db/src/buffers/session-buffer.ts
View File

@@ -109,6 +109,12 @@ export class SessionBuffer extends BaseBuffer {
newSession.profile_id = event.profile_id;
}
if (event.groups) {
newSession.groups = [
...new Set([...newSession.groups, ...event.groups]),
];
}
return [newSession, oldSession];
}
@@ -119,6 +125,7 @@ export class SessionBuffer extends BaseBuffer {
profile_id: event.profile_id,
project_id: event.project_id,
device_id: event.device_id,
groups: event.groups,
created_at: event.created_at,
ended_at: event.created_at,
event_count: event.name === 'screen_view' ? 0 : 1,

9
packages/db/src/clickhouse/client.ts
View File

@@ -61,6 +61,7 @@ export const TABLE_NAMES = {
gsc_daily: 'gsc_daily',
gsc_pages_daily: 'gsc_pages_daily',
gsc_queries_daily: 'gsc_queries_daily',
groups: 'groups',
};
/**
@@ -298,3 +299,11 @@ const ROLLUP_DATE_PREFIX = '1970-01-01';
export function isClickhouseDefaultMinDate(date: string): boolean {
return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');
}
export function toNullIfDefaultMinDate(date?: string | null): Date | null {
if (!date) {
return null;
}
return isClickhouseDefaultMinDate(date)
? null
: convertClickhouseDateToJs(date);
}

12
packages/db/src/clickhouse/query-builder.ts
View File

@@ -66,6 +66,7 @@ export class Query<T = any> {
alias?: string;
}[] = [];
private _skipNext = false;
private _rawJoins: string[] = [];
private _fill?: {
from: string | Date;
to: string | Date;
@@ -339,6 +340,12 @@ export class Query<T = any> {
return this.joinWithType('CROSS', table, '', alias);
}
rawJoin(sql: string): this {
if (this._skipNext) return this;
this._rawJoins.push(sql);
return this;
}
private joinWithType(
type: JoinType,
table: string | Expression | Query,
@@ -426,6 +433,10 @@ export class Query<T = any> {
`${join.type} JOIN ${join.table instanceof Query ? `(${join.table.toSQL()})` : join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`
);
});
// Add raw joins (e.g. ARRAY JOIN)
this._rawJoins.forEach((join) => {
parts.push(join);
});
}
// WHERE
@@ -604,6 +615,7 @@ export class Query<T = any> {
// Merge JOINS
this._joins = [...this._joins, ...query._joins];
this._rawJoins = [...this._rawJoins, ...query._rawJoins];
// Merge settings
this._settings = { ...this._settings, ...query._settings };

16
packages/db/src/engine/format.ts
View File

@@ -26,7 +26,7 @@ export function format(
}>,
includeAlphaIds: boolean,
previousSeries: ConcreteSeries[] | null = null,
limit: number | undefined = undefined,
limit: number | undefined = undefined
): FinalChart {
const series = concreteSeries.map((cs) => {
// Find definition for this series
@@ -70,7 +70,7 @@ export function format(
const previousSerie = previousSeries?.find(
(ps) =>
ps.definitionIndex === cs.definitionIndex &&
ps.name.slice(1).join(':::') === cs.name.slice(1).join(':::'),
ps.name.slice(1).join(':::') === cs.name.slice(1).join(':::')
);
return {
@@ -89,24 +89,24 @@ export function format(
previous: {
sum: getPreviousMetric(
metrics.sum,
sum(previousSerie.data.map((d) => d.count)),
sum(previousSerie.data.map((d) => d.count))
),
average: getPreviousMetric(
metrics.average,
round(average(previousSerie.data.map((d) => d.count)), 2),
round(average(previousSerie.data.map((d) => d.count)), 2)
),
min: getPreviousMetric(
metrics.min,
min(previousSerie.data.map((d) => d.count)),
min(previousSerie.data.map((d) => d.count))
),
max: getPreviousMetric(
metrics.max,
max(previousSerie.data.map((d) => d.count)),
max(previousSerie.data.map((d) => d.count))
),
count: getPreviousMetric(
metrics.count ?? 0,
previousSerie.data.find((item) => !!item.total_count)
?.total_count ?? null,
?.total_count ?? null
),
},
}
@@ -118,7 +118,7 @@ export function format(
previous: previousSerie?.data[index]
? getPreviousMetric(
item.count,
previousSerie.data[index]?.count ?? null,
previousSerie.data[index]?.count ?? null
)
: undefined,
})),

19
packages/db/src/engine/index.ts
View File

@@ -1,5 +1,5 @@
import { getPreviousMetric, groupByLabels } from '@openpanel/common';
import type { ISerieDataItem } from '@openpanel/common';
import { groupByLabels } from '@openpanel/common';
import { alphabetIds } from '@openpanel/constants';
import type {
FinalChart,
@@ -33,7 +33,7 @@ export async function executeChart(input: IReportInput): Promise<FinalChart> {
// Handle subscription end date limit
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
normalized.endDate,
normalized.endDate
);
if (endDate) {
normalized.endDate = endDate;
@@ -73,6 +73,7 @@ export async function executeChart(input: IReportInput): Promise<FinalChart> {
executionPlan.definitions,
includeAlphaIds,
previousSeries,
normalized.limit
);
return response;
@@ -83,7 +84,7 @@ export async function executeChart(input: IReportInput): Promise<FinalChart> {
* Executes a simplified pipeline: normalize -> fetch aggregate -> format
*/
export async function executeAggregateChart(
input: IReportInput,
input: IReportInput
): Promise<FinalChart> {
// Stage 1: Normalize input
const normalized = await normalize(input);
@@ -91,7 +92,7 @@ export async function executeAggregateChart(
// Handle subscription end date limit
const endDate = await getOrganizationSubscriptionChartEndDate(
input.projectId,
normalized.endDate,
normalized.endDate
);
if (endDate) {
normalized.endDate = endDate;
@@ -137,7 +138,7 @@ export async function executeAggregateChart(
getAggregateChartSql(queryInput),
{
session_timezone: timezone,
},
}
);
// Fallback: if no results with breakdowns, try without breakdowns
@@ -149,7 +150,7 @@ export async function executeAggregateChart(
}),
{
session_timezone: timezone,
},
}
);
}
@@ -262,7 +263,7 @@ export async function executeAggregateChart(
getAggregateChartSql(queryInput),
{
session_timezone: timezone,
},
}
);
if (queryResult.length === 0 && normalized.breakdowns.length > 0) {
@@ -273,7 +274,7 @@ export async function executeAggregateChart(
}),
{
session_timezone: timezone,
},
}
);
}
@@ -344,7 +345,7 @@ export async function executeAggregateChart(
normalized.series,
includeAlphaIds,
previousSeries,
normalized.limit,
normalized.limit
);
return response;

499
packages/db/src/services/chart.service.ts
View File

@@ -1,20 +1,19 @@
import sqlstring from 'sqlstring';
/** biome-ignore-all lint/style/useDefaultSwitchClause: switch cases are exhaustive by design */
import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common';
import type {
IChartEventFilter,
IReportInput,
IChartRange,
IGetChartDataInput,
IReportInput,
} from '@openpanel/validation';
import { TABLE_NAMES, formatClickhouseDate } from '../clickhouse/client';
import sqlstring from 'sqlstring';
import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
export function transformPropertyKey(property: string) {
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
property.startsWith(`${pattern}.`)
);
if (!match) {
@@ -32,21 +31,91 @@ export function transformPropertyKey(property: string) {
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
export function getSelectPropertyKey(property: string) {
// Returns a SQL expression for a group property via the _g JOIN alias
// property format: "group.name", "group.type", "group.properties.plan"
export function getGroupPropertySql(property: string): string {
const withoutPrefix = property.replace(/^group\./, '');
if (withoutPrefix === 'name') {
return '_g.name';
}
if (withoutPrefix === 'type') {
return '_g.type';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `_g.properties[${sqlstring.escape(propKey)}]`;
}
return '_group_id';
}
// Returns the SELECT expression when querying the groups table directly (no join alias).
// Use for fetching distinct values for group.* properties.
export function getGroupPropertySelect(property: string): string {
const withoutPrefix = property.replace(/^group\./, '');
if (withoutPrefix === 'name') {
return 'name';
}
if (withoutPrefix === 'type') {
return 'type';
}
if (withoutPrefix === 'id') {
return 'id';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `properties[${sqlstring.escape(propKey)}]`;
}
return 'id';
}
// Returns the SELECT expression when querying the profiles table directly (no join alias).
// Use for fetching distinct values for profile.* properties.
export function getProfilePropertySelect(property: string): string {
const withoutPrefix = property.replace(/^profile\./, '');
if (withoutPrefix === 'id') {
return 'id';
}
if (withoutPrefix === 'first_name') {
return 'first_name';
}
if (withoutPrefix === 'last_name') {
return 'last_name';
}
if (withoutPrefix === 'email') {
return 'email';
}
if (withoutPrefix === 'avatar') {
return 'avatar';
}
if (withoutPrefix.startsWith('properties.')) {
const propKey = withoutPrefix.replace(/^properties\./, '');
return `properties[${sqlstring.escape(propKey)}]`;
}
return 'id';
}
export function getSelectPropertyKey(property: string, projectId?: string) {
if (property === 'has_profile') {
return `if(profile_id != device_id, 'true', 'false')`;
}
// Handle group properties — requires ARRAY JOIN + _g JOIN to be present in query
if (property.startsWith('group.') && projectId) {
return getGroupPropertySql(property);
}
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
property.startsWith(`${pattern}.`)
);
if (!match) return property;
if (!match) {
return property;
}
if (property.includes('*')) {
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${sqlstring.escape(
transformPropertyKey(property),
transformPropertyKey(property)
)})))`;
}
@@ -60,9 +129,7 @@ export function getChartSql({
startDate,
endDate,
projectId,
limit,
timezone,
chartType,
}: IGetChartDataInput & { timezone: string }) {
const {
sb,
@@ -78,22 +145,43 @@ export function getChartSql({
with: addCte,
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
sb.where.eventName = `e.name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
filter.name.startsWith('profile.'),
filter.name.startsWith('profile.')
);
const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
breakdown.name.startsWith('profile.'),
breakdown.name.startsWith('profile.')
);
const anyFilterOnGroup = event.filters.some((filter) =>
filter.name.startsWith('group.')
);
const anyBreakdownOnGroup = breakdowns.some((breakdown) =>
breakdown.name.startsWith('group.')
);
const anyMetricOnGroup = !!event.property?.startsWith('group.');
const needsGroupArrayJoin =
anyFilterOnGroup ||
anyBreakdownOnGroup ||
anyMetricOnGroup ||
event.segment === 'group';
if (needsGroupArrayJoin) {
addCte(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_table = 'LEFT ANY JOIN _g ON _g.id = _group_id';
}
// Build WHERE clause without the bar filter (for use in subqueries and CTEs)
// Define this early so we can use it in CTE definitions
@@ -178,8 +266,8 @@ export function getChartSql({
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`
);
// Use the CTE reference in the main query
@@ -225,31 +313,11 @@ export function getChartSql({
sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
}
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
.map((b) => getSelectPropertyKey(b.name))
.join(', ');
// Add top_breakdowns CTE using the builder
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.select[key] =
`${getSelectPropertyKey(breakdown.name, projectId)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -261,6 +329,10 @@ export function getChartSql({
sb.select.count = 'countDistinct(session_id) as count';
}
if (event.segment === 'group') {
sb.select.count = 'countDistinct(_group_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
@@ -287,9 +359,9 @@ export function getChartSql({
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} e ${getJoins()} WHERE ${join(
sb.where,
' AND ',
' AND '
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
@@ -303,41 +375,52 @@ export function getChartSql({
}
// Note: The profile CTE (if it exists) is available in subqueries, so we can reference it directly
const subqueryGroupJoins = needsGroupArrayJoin
? 'ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id '
: '';
if (breakdowns.length > 0) {
// Match breakdown properties in subquery with outer query's grouped values
// Since outer query groups by label_X, we reference those in the correlation
const breakdownMatches = breakdowns
// Pre-compute unique counts per breakdown group in a CTE, then JOIN it.
// We can't use a correlated subquery because:
// 1. ClickHouse expands label_X aliases to their underlying expressions,
// which resolve in the subquery's scope, making the condition a tautology.
// 2. Correlated subqueries aren't supported on distributed/remote tables.
const ucSelectParts: string[] = breakdowns.map((breakdown, index) => {
const propertyKey = getSelectPropertyKey(breakdown.name, projectId);
return `${propertyKey} as _uc_label_${index + 1}`;
});
ucSelectParts.push('uniq(profile_id) as total_count');
const ucGroupByParts = breakdowns.map(
(_, index) => `_uc_label_${index + 1}`
);
const ucWhere = getWhereWithoutBar();
addCte(
'_uc',
`SELECT ${ucSelectParts.join(', ')} FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${ucWhere} GROUP BY ${ucGroupByParts.join(', ')}`
);
const ucJoinConditions = breakdowns
.map((b, index) => {
const propertyKey = getSelectPropertyKey(b.name);
// Correlate: match the property expression with outer query's label_X value
// ClickHouse allows referencing outer query columns in correlated subqueries
return `${propertyKey} = label_${index + 1}`;
const propertyKey = getSelectPropertyKey(b.name, projectId);
return `_uc._uc_label_${index + 1} = ${propertyKey}`;
})
.join(' AND ');
// Build WHERE clause for subquery - replace table alias and keep profile CTE reference
const subqueryWhere = getWhereWithoutBar()
.replace(/\be\./g, 'e2.')
.replace(/\bprofile\./g, 'profile.');
sb.select.total_unique_count = `(
SELECT uniq(profile_id)
FROM ${TABLE_NAMES.events} e2
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
AND ${breakdownMatches}
) as total_count`;
sb.joins.unique_counts = `LEFT ANY JOIN _uc ON ${ucJoinConditions}`;
sb.select.total_unique_count = 'any(_uc.total_count) as total_count';
} else {
// No breakdowns: calculate unique count across all data
// Build WHERE clause for subquery - replace table alias and keep profile CTE reference
const subqueryWhere = getWhereWithoutBar()
.replace(/\be\./g, 'e2.')
.replace(/\bprofile\./g, 'profile.');
const ucWhere = getWhereWithoutBar();
sb.select.total_unique_count = `(
SELECT uniq(profile_id)
FROM ${TABLE_NAMES.events} e2
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${subqueryWhere}
) as total_count`;
addCte(
'_uc',
`SELECT uniq(profile_id) as total_count FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${ucWhere}`
);
sb.select.total_unique_count =
'(SELECT total_count FROM _uc) as total_count';
}
const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
@@ -359,31 +442,43 @@ export function getAggregateChartSql({
}) {
const { sb, join, getJoins, with: addCte, getSql } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
sb.where.eventName = `e.name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
const anyFilterOnProfile = event.filters.some((filter) =>
filter.name.startsWith('profile.'),
filter.name.startsWith('profile.')
);
const anyBreakdownOnProfile = breakdowns.some((breakdown) =>
breakdown.name.startsWith('profile.'),
breakdown.name.startsWith('profile.')
);
const anyFilterOnGroup = event.filters.some((filter) =>
filter.name.startsWith('group.')
);
const anyBreakdownOnGroup = breakdowns.some((breakdown) =>
breakdown.name.startsWith('group.')
);
const anyMetricOnGroup = !!event.property?.startsWith('group.');
const needsGroupArrayJoin =
anyFilterOnGroup ||
anyBreakdownOnGroup ||
anyMetricOnGroup ||
event.segment === 'group';
// Build WHERE clause without the bar filter (for use in subqueries and CTEs)
const getWhereWithoutBar = () => {
const whereWithoutBar = { ...sb.where };
delete whereWithoutBar.bar;
return Object.keys(whereWithoutBar).length
? `WHERE ${join(whereWithoutBar, ' AND ')}`
: '';
};
if (needsGroupArrayJoin) {
addCte(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_table = 'LEFT ANY JOIN _g ON _g.id = _group_id';
}
// Collect all profile fields used in filters and breakdowns
const getProfileFields = () => {
@@ -455,8 +550,8 @@ export function getAggregateChartSql({
addCte(
'profile',
`SELECT ${selectFields.join(', ')}
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`,
FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}`
);
sb.joins.profiles = profilesJoinRef;
@@ -475,31 +570,12 @@ export function getAggregateChartSql({
// Use startDate as the date value since we're aggregating across the entire range
sb.select.date = `${sqlstring.escape(startDate)} as date`;
// Use CTE to define top breakdown values once, then reference in WHERE clause
if (breakdowns.length > 0 && limit) {
const breakdownSelects = breakdowns
.map((b) => getSelectPropertyKey(b.name))
.join(', ');
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
LIMIT ${limit}`,
);
// Filter main query to only include top breakdown values
sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;
}
// Add breakdowns to SELECT and GROUP BY
breakdowns.forEach((breakdown, index) => {
// Breakdowns start at label_1 (label_0 is reserved for event name)
const key = `label_${index + 1}`;
sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
sb.select[key] =
`${getSelectPropertyKey(breakdown.name, projectId)} as ${key}`;
sb.groupBy[key] = `${key}`;
});
@@ -518,6 +594,10 @@ export function getAggregateChartSql({
sb.select.count = 'countDistinct(session_id) as count';
}
if (event.segment === 'group') {
sb.select.count = 'countDistinct(_group_id) as count';
}
if (event.segment === 'user_average') {
sb.select.count =
'COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count';
@@ -531,7 +611,7 @@ export function getAggregateChartSql({
}[event.segment as string];
if (mathFunction && event.property) {
const propertyKey = getSelectPropertyKey(event.property);
const propertyKey = getSelectPropertyKey(event.property, projectId);
if (isNumericColumn(event.property)) {
sb.select.count = `${mathFunction}(${propertyKey}) as count`;
@@ -544,9 +624,9 @@ export function getAggregateChartSql({
if (event.segment === 'one_event_per_user') {
sb.from = `(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} ${getJoins()} WHERE ${join(
SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} e ${getJoins()} WHERE ${join(
sb.where,
' AND ',
' AND '
)}
ORDER BY profile_id, created_at DESC
) as subQuery`;
@@ -579,7 +659,10 @@ function isNumericColumn(columnName: string): boolean {
return numericColumns.includes(columnName);
}
export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
export function getEventFiltersWhereClause(
filters: IChartEventFilter[],
projectId?: string
) {
const where: Record<string, string> = {};
filters.forEach((filter, index) => {
const id = `f${index}`;
@@ -602,6 +685,67 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
return;
}
// Handle group. prefixed filters (requires ARRAY JOIN + _g JOIN in query)
if (name.startsWith('group.') && projectId) {
const whereFrom = getGroupPropertySql(name);
switch (operator) {
case 'is': {
if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] =
`${whereFrom} IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')})`;
}
break;
}
case 'isNot': {
if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] =
`${whereFrom} NOT IN (${value.map((val) => sqlstring.escape(String(val).trim())).join(', ')})`;
}
break;
}
case 'contains': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'doesNotContain': {
where[id] =
`(${value.map((val) => `${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'startsWith': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`).join(' OR ')})`;
break;
}
case 'endsWith': {
where[id] =
`(${value.map((val) => `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`).join(' OR ')})`;
break;
}
case 'isNull': {
where[id] = `(${whereFrom} = '' OR ${whereFrom} IS NULL)`;
break;
}
case 'isNotNull': {
where[id] = `(${whereFrom} != '' AND ${whereFrom} IS NOT NULL)`;
break;
}
case 'regex': {
where[id] =
`(${value.map((val) => `match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`).join(' OR ')})`;
break;
}
}
return;
}
if (
name.startsWith('properties.') ||
name.startsWith('profile.properties.')
@@ -616,15 +760,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x = ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
if (value.length === 1) {
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
where[id] = `${whereFrom} IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
}
@@ -633,15 +775,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x != ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
if (value.length === 1) {
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} NOT IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
where[id] = `${whereFrom} NOT IN (${value
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
}
@@ -649,15 +789,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -668,14 +807,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -685,14 +824,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
(val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')})`;
}
@@ -702,14 +841,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map(
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')})`;
}
@@ -724,7 +863,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`,
`match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -752,14 +891,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -770,14 +909,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -788,14 +927,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -806,14 +945,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `arrayExists(x -> ${value
.map(
(val) =>
`toFloat64OrZero(x) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(x) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`toFloat64OrZero(${whereFrom}) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64OrZero(${whereFrom}) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
}
@@ -856,7 +995,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -865,7 +1004,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
`${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -874,7 +1013,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
`${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`
)
.join(' OR ')})`;
break;
@@ -883,7 +1022,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`
)
.join(' OR ')})`;
break;
@@ -892,7 +1031,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
`match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`
)
.join(' OR ')})`;
break;
@@ -902,7 +1041,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) > toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) > toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
@@ -917,7 +1056,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) < toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) < toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
@@ -932,13 +1071,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) >= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) >= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
where[id] = `(${value
.map(
(val) => `${name} >= ${sqlstring.escape(String(val).trim())}`,
(val) => `${name} >= ${sqlstring.escape(String(val).trim())}`
)
.join(' OR ')})`;
}
@@ -949,13 +1088,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`toFloat64(${name}) <= toFloat64(${sqlstring.escape(String(val).trim())})`,
`toFloat64(${name}) <= toFloat64(${sqlstring.escape(String(val).trim())})`
)
.join(' OR ')})`;
} else {
where[id] = `(${value
.map(
(val) => `${name} <= ${sqlstring.escape(String(val).trim())}`,
(val) => `${name} <= ${sqlstring.escape(String(val).trim())}`
)
.join(' OR ')})`;
}
@@ -974,15 +1113,15 @@ export function getChartStartEndDate(
endDate,
range,
}: Pick<IReportInput, 'endDate' | 'startDate' | 'range'>,
timezone: string,
timezone: string
) {
if (startDate && endDate) {
return { startDate: startDate, endDate: endDate };
return { startDate, endDate };
}
const ranges = getDatesFromRange(range, timezone);
if (!startDate && endDate) {
return { startDate: ranges.startDate, endDate: endDate };
return { startDate: ranges.startDate, endDate };
}
return ranges;
@@ -1002,8 +1141,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1018,8 +1157,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1035,8 +1174,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.endOf('day')
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1053,8 +1192,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1071,8 +1210,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1089,8 +1228,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1106,8 +1245,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1124,8 +1263,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1141,8 +1280,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1152,8 +1291,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1170,8 +1309,8 @@ export function getDatesFromRange(range: IChartRange, timezone: string) {
.toFormat('yyyy-MM-dd HH:mm:ss');
return {
startDate: startDate,
endDate: endDate,
startDate,
endDate,
};
}
@@ -1183,7 +1322,7 @@ export function getChartPrevStartEndDate({
endDate: string;
}) {
let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff(
DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'),
DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss')
);
// this will make sure our start and end date's are correct

32
packages/db/src/services/conversion.service.ts
View File

@@ -31,14 +31,14 @@ export class ConversionService {
const funnelWindow = funnelOptions?.funnelWindow ?? 24;
const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';
const breakdownExpressions = breakdowns.map(
(b) => getSelectPropertyKey(b.name),
(b) => getSelectPropertyKey(b.name, projectId),
);
const breakdownSelects = breakdownExpressions.map(
(expr, index) => `${expr} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);
// Check if any breakdown uses profile fields and build profile JOIN if needed
// Check if any breakdown or filter uses profile fields
const profileBreakdowns = breakdowns.filter((b) =>
b.name.startsWith('profile.'),
);
@@ -71,6 +71,15 @@ export class ConversionService {
const events = onlyReportEvents(series);
// Check if any breakdown or filter uses group fields
const anyBreakdownOnGroup = breakdowns.some((b) =>
b.name.startsWith('group.'),
);
const anyFilterOnGroup = events.some((e) =>
e.filters?.some((f) => f.name.startsWith('group.')),
);
const needsGroupArrayJoin = anyBreakdownOnGroup || anyFilterOnGroup;
if (events.length !== 2) {
throw new Error('events must be an array of two events');
}
@@ -82,21 +91,25 @@ export class ConversionService {
const eventA = events[0]!;
const eventB = events[1]!;
const whereA = Object.values(
getEventFiltersWhereClause(eventA.filters),
getEventFiltersWhereClause(eventA.filters, projectId),
).join(' AND ');
const whereB = Object.values(
getEventFiltersWhereClause(eventB.filters),
getEventFiltersWhereClause(eventB.filters, projectId),
).join(' AND ');
const funnelWindowSeconds = funnelWindow * 3600;
// Build funnel conditions
const conditionA = whereA
? `(name = '${eventA.name}' AND ${whereA})`
: `name = '${eventA.name}'`;
? `(events.name = '${eventA.name}' AND ${whereA})`
: `events.name = '${eventA.name}'`;
const conditionB = whereB
? `(name = '${eventB.name}' AND ${whereB})`
: `name = '${eventB.name}'`;
? `(events.name = '${eventB.name}' AND ${whereB})`
: `events.name = '${eventB.name}'`;
const groupJoin = needsGroupArrayJoin
? `ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`
: '';
// Use windowFunnel approach - single scan, no JOIN
const query = clix(this.client, timezone)
@@ -126,8 +139,9 @@ export class ConversionService {
) as steps
FROM ${TABLE_NAMES.events}
${profileJoin}
${groupJoin}
WHERE project_id = '${projectId}'
AND name IN ('${eventA.name}', '${eventB.name}')
AND events.name IN ('${eventA.name}', '${eventB.name}')
AND created_at BETWEEN toDateTime('${startDate}') AND toDateTime('${endDate}')
GROUP BY ${group}${breakdownExpressions.length ? `, ${breakdownExpressions.join(', ')}` : ''})
`),

56
packages/db/src/services/event.service.ts
View File

@@ -32,14 +32,14 @@ export type IImportedEvent = Omit<
properties: Record<string, unknown>;
};
export type IServicePage = {
export interface IServicePage {
path: string;
count: number;
project_id: string;
first_seen: string;
title: string;
origin: string;
};
}
export interface IClickhouseBotEvent {
id: string;
@@ -92,6 +92,7 @@ export interface IClickhouseEvent {
sdk_name: string;
sdk_version: string;
revenue?: number;
groups: string[];
// They do not exist here. Just make ts happy for now
profile?: IServiceProfile;
@@ -143,6 +144,7 @@ export function transformSessionToEvent(
importedAt: undefined,
sdkName: undefined,
sdkVersion: undefined,
groups: [],
};
}
@@ -179,6 +181,7 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
sdkVersion: event.sdk_version,
profile: event.profile,
revenue: event.revenue,
groups: event.groups ?? [],
};
}
@@ -227,6 +230,7 @@ export interface IServiceEvent {
sdkName: string | undefined;
sdkVersion: string | undefined;
revenue?: number;
groups: string[];
}
type SelectHelper<T> = {
@@ -331,6 +335,7 @@ export async function getEvents(
projectId,
isExternal: false,
properties: {},
groups: [],
};
}
}
@@ -386,6 +391,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
sdk_name: payload.sdkName ?? '',
sdk_version: payload.sdkVersion ?? '',
revenue: payload.revenue,
groups: payload.groups ?? [],
};
const promises = [sessionBuffer.add(event), eventBuffer.add(event)];
@@ -434,6 +440,7 @@ export interface GetEventListOptions {
projectId: string;
profileId?: string;
sessionId?: string;
groupId?: string;
take: number;
cursor?: number | Date;
events?: string[] | null;
@@ -452,6 +459,7 @@ export async function getEventList(options: GetEventListOptions) {
projectId,
profileId,
sessionId,
groupId,
events,
filters,
startDate,
@@ -589,6 +597,10 @@ export async function getEventList(options: GetEventListOptions) {
sb.select.revenue = 'revenue';
}
if (select.groups) {
sb.select.groups = 'groups';
}
if (profileId) {
sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND device_id != '' AND profile_id = ${sqlstring.escape(profileId)} group by did) OR profile_id = ${sqlstring.escape(profileId)})`;
}
@@ -597,6 +609,10 @@ export async function getEventList(options: GetEventListOptions) {
sb.where.sessionId = `session_id = ${sqlstring.escape(sessionId)}`;
}
if (groupId) {
sb.where.groupId = `has(groups, ${sqlstring.escape(groupId)})`;
}
if (startDate && endDate) {
sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
}
@@ -611,7 +627,7 @@ export async function getEventList(options: GetEventListOptions) {
if (filters) {
sb.where = {
...sb.where,
...getEventFiltersWhereClause(filters),
...getEventFiltersWhereClause(filters, projectId),
};
// Join profiles table if any filter uses profile fields
@@ -622,6 +638,13 @@ export async function getEventList(options: GetEventListOptions) {
if (profileFilters.length > 0) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0])).join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
// Join groups table if any filter uses group fields
const groupFilters = filters.filter((f) => f.name.startsWith('group.'));
if (groupFilters.length > 0) {
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
}
}
sb.orderBy.created_at = 'created_at DESC, id ASC';
@@ -653,6 +676,7 @@ export async function getEventList(options: GetEventListOptions) {
export async function getEventsCount({
projectId,
profileId,
groupId,
events,
filters,
startDate,
@@ -664,6 +688,10 @@ export async function getEventsCount({
sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
}
if (groupId) {
sb.where.groupId = `has(groups, ${sqlstring.escape(groupId)})`;
}
if (startDate && endDate) {
sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
}
@@ -678,7 +706,7 @@ export async function getEventsCount({
if (filters) {
sb.where = {
...sb.where,
...getEventFiltersWhereClause(filters),
...getEventFiltersWhereClause(filters, projectId),
};
// Join profiles table if any filter uses profile fields
@@ -689,6 +717,13 @@ export async function getEventsCount({
if (profileFilters.length > 0) {
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0])).join(', ')} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
// Join groups table if any filter uses group fields
const groupFilters = filters.filter((f) => f.name.startsWith('group.'));
if (groupFilters.length > 0) {
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
}
}
const res = await chQuery<{ count: number }>(
@@ -1052,8 +1087,19 @@ class EventService {
}
if (filters) {
q.rawWhere(
Object.values(getEventFiltersWhereClause(filters)).join(' AND ')
Object.values(
getEventFiltersWhereClause(filters, projectId)
).join(' AND ')
);
const groupFilters = filters.filter((f) =>
f.name.startsWith('group.')
);
if (groupFilters.length > 0) {
q.rawJoin('ARRAY JOIN groups AS _group_id');
q.rawJoin(
`LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`
);
}
}
},
session: (q) => {

33
packages/db/src/services/funnel.service.ts
View File

@@ -34,11 +34,11 @@ export class FunnelService {
return group === 'profile_id' ? 'profile_id' : 'session_id';
}
getFunnelConditions(events: IChartEvent[] = []): string[] {
getFunnelConditions(events: IChartEvent[] = [], projectId?: string): string[] {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.name = `events.name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
}
@@ -71,7 +71,7 @@ export class FunnelService {
additionalGroupBy?: string[];
group?: 'session_id' | 'profile_id';
}) {
const funnels = this.getFunnelConditions(eventSeries);
const funnels = this.getFunnelConditions(eventSeries, projectId);
const primaryKey = group === 'profile_id' ? 'profile_id' : 'session_id';
return clix(this.client, timezone)
@@ -90,7 +90,7 @@ export class FunnelService {
clix.datetime(endDate, 'toDateTime'),
])
.where(
'name',
'events.name',
'IN',
eventSeries.map((e) => e.name),
)
@@ -236,10 +236,18 @@ export class FunnelService {
const anyBreakdownOnProfile = breakdowns.some((b) =>
b.name.startsWith('profile.'),
);
const anyFilterOnGroup = eventSeries.some((e) =>
e.filters?.some((f) => f.name.startsWith('group.')),
);
const anyBreakdownOnGroup = breakdowns.some((b) =>
b.name.startsWith('group.'),
);
const needsGroupArrayJoin =
anyFilterOnGroup || anyBreakdownOnGroup || funnelGroup === 'group';
// Create the funnel CTE (session-level)
const breakdownSelects = breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
(b, index) => `${getSelectPropertyKey(b.name, projectId)} as b_${index}`,
);
const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);
@@ -277,8 +285,21 @@ export class FunnelService {
);
}
if (needsGroupArrayJoin) {
funnelCte.rawJoin('ARRAY JOIN groups AS _group_id');
funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');
}
// Base funnel query with CTEs
const funnelQuery = clix(this.client, timezone);
if (needsGroupArrayJoin) {
funnelQuery.with(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`,
);
}
funnelQuery.with('session_funnel', funnelCte);
// windowFunnel is computed per the primary key (profile_id or session_id),

363
packages/db/src/services/group.service.ts Normal file
View File

@@ -0,0 +1,363 @@
import { toDots } from '@openpanel/common';
import sqlstring from 'sqlstring';
import {
ch,
chQuery,
formatClickhouseDate,
TABLE_NAMES,
} from '../clickhouse/client';
import type { IServiceProfile } from './profile.service';
import { getProfiles } from './profile.service';
export type IServiceGroup = {
id: string;
projectId: string;
type: string;
name: string;
properties: Record<string, unknown>;
createdAt: Date;
updatedAt: Date;
};
export type IServiceUpsertGroup = {
id: string;
projectId: string;
type: string;
name: string;
properties?: Record<string, unknown>;
};
type IClickhouseGroup = {
project_id: string;
id: string;
type: string;
name: string;
properties: Record<string, string>;
created_at: string;
version: string;
};
function transformGroup(row: IClickhouseGroup): IServiceGroup {
return {
id: row.id,
projectId: row.project_id,
type: row.type,
name: row.name,
properties: row.properties,
createdAt: new Date(row.created_at),
updatedAt: new Date(Number(row.version)),
};
}
async function writeGroupToCh(
group: {
id: string;
projectId: string;
type: string;
name: string;
properties: Record<string, string>;
createdAt?: Date;
},
deleted = 0
) {
await ch.insert({
format: 'JSONEachRow',
table: TABLE_NAMES.groups,
values: [
{
project_id: group.projectId,
id: group.id,
type: group.type,
name: group.name,
properties: group.properties,
created_at: formatClickhouseDate(group.createdAt ?? new Date()),
version: Date.now(),
deleted,
},
],
});
}
export async function upsertGroup(input: IServiceUpsertGroup) {
const existing = await getGroupById(input.id, input.projectId);
await writeGroupToCh({
id: input.id,
projectId: input.projectId,
type: input.type,
name: input.name,
properties: toDots({
...(existing?.properties ?? {}),
...(input.properties ?? {}),
}),
createdAt: existing?.createdAt,
});
}
export async function getGroupById(
id: string,
projectId: string
): Promise<IServiceGroup | null> {
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id = ${sqlstring.escape(id)}
AND deleted = 0
`);
return rows[0] ? transformGroup(rows[0]) : null;
}
export async function getGroupList({
projectId,
cursor,
take,
search,
type,
}: {
projectId: string;
cursor?: number;
take: number;
search?: string;
type?: string;
}): Promise<IServiceGroup[]> {
const conditions = [
`project_id = ${sqlstring.escape(projectId)}`,
'deleted = 0',
...(type ? [`type = ${sqlstring.escape(type)}`] : []),
...(search
? [
`(name ILIKE ${sqlstring.escape(`%${search}%`)} OR id ILIKE ${sqlstring.escape(`%${search}%`)})`,
]
: []),
];
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE ${conditions.join(' AND ')}
ORDER BY created_at DESC
LIMIT ${take}
OFFSET ${Math.max(0, (cursor ?? 0) * take)}
`);
return rows.map(transformGroup);
}
export async function getGroupListCount({
projectId,
type,
search,
}: {
projectId: string;
type?: string;
search?: string;
}): Promise<number> {
const conditions = [
`project_id = ${sqlstring.escape(projectId)}`,
'deleted = 0',
...(type ? [`type = ${sqlstring.escape(type)}`] : []),
...(search
? [
`(name ILIKE ${sqlstring.escape(`%${search}%`)} OR id ILIKE ${sqlstring.escape(`%${search}%`)})`,
]
: []),
];
const rows = await chQuery<{ count: number }>(`
SELECT count() as count
FROM ${TABLE_NAMES.groups} FINAL
WHERE ${conditions.join(' AND ')}
`);
return rows[0]?.count ?? 0;
}
export async function getGroupTypes(projectId: string): Promise<string[]> {
const rows = await chQuery<{ type: string }>(`
SELECT DISTINCT type
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND deleted = 0
`);
return rows.map((r) => r.type);
}
export async function createGroup(input: IServiceUpsertGroup) {
await upsertGroup(input);
return getGroupById(input.id, input.projectId);
}
export async function updateGroup(
id: string,
projectId: string,
data: { type?: string; name?: string; properties?: Record<string, unknown> }
) {
const existing = await getGroupById(id, projectId);
if (!existing) {
throw new Error(`Group ${id} not found`);
}
const mergedProperties = {
...(existing.properties ?? {}),
...(data.properties ?? {}),
};
const normalizedProperties = toDots(
mergedProperties as Record<string, unknown>
);
const updated = {
id,
projectId,
type: data.type ?? existing.type,
name: data.name ?? existing.name,
properties: normalizedProperties,
createdAt: existing.createdAt,
};
await writeGroupToCh(updated);
return { ...existing, ...updated };
}
export async function deleteGroup(id: string, projectId: string) {
const existing = await getGroupById(id, projectId);
if (!existing) {
throw new Error(`Group ${id} not found`);
}
await writeGroupToCh(
{
id,
projectId,
type: existing.type,
name: existing.name,
properties: existing.properties as Record<string, string>,
createdAt: existing.createdAt,
},
1
);
return existing;
}
export async function getGroupPropertyKeys(
projectId: string
): Promise<string[]> {
const rows = await chQuery<{ key: string }>(`
SELECT DISTINCT arrayJoin(mapKeys(properties)) as key
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND deleted = 0
`);
return rows.map((r) => r.key).sort();
}
export type IServiceGroupStats = {
groupId: string;
memberCount: number;
lastActiveAt: Date | null;
};
export async function getGroupStats(
projectId: string,
groupIds: string[]
): Promise<Map<string, IServiceGroupStats>> {
if (groupIds.length === 0) {
return new Map();
}
const rows = await chQuery<{
group_id: string;
member_count: number;
last_active_at: string;
}>(`
SELECT
g AS group_id,
uniqExact(profile_id) AS member_count,
max(created_at) AS last_active_at
FROM ${TABLE_NAMES.events}
ARRAY JOIN groups AS g
WHERE project_id = ${sqlstring.escape(projectId)}
AND g IN (${groupIds.map((id) => sqlstring.escape(id)).join(',')})
AND profile_id != device_id
GROUP BY g
`);
return new Map(
rows.map((r) => [
r.group_id,
{
groupId: r.group_id,
memberCount: r.member_count,
lastActiveAt: r.last_active_at ? new Date(r.last_active_at) : null,
},
])
);
}
export async function getGroupsByIds(
projectId: string,
ids: string[]
): Promise<IServiceGroup[]> {
if (ids.length === 0) {
return [];
}
const rows = await chQuery<IClickhouseGroup>(`
SELECT project_id, id, type, name, properties, created_at, version
FROM ${TABLE_NAMES.groups} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
AND id IN (${ids.map((id) => sqlstring.escape(id)).join(',')})
AND deleted = 0
`);
return rows.map(transformGroup);
}
export async function getGroupMemberProfiles({
projectId,
groupId,
cursor,
take,
search,
}: {
projectId: string;
groupId: string;
cursor?: number;
take: number;
search?: string;
}): Promise<{ data: IServiceProfile[]; count: number }> {
const offset = Math.max(0, (cursor ?? 0) * take);
const searchCondition = search?.trim()
? `AND (email ILIKE ${sqlstring.escape(`%${search.trim()}%`)} OR first_name ILIKE ${sqlstring.escape(`%${search.trim()}%`)} OR last_name ILIKE ${sqlstring.escape(`%${search.trim()}%`)})`
: '';
// count() OVER () is evaluated after JOINs/WHERE but before LIMIT,
// so we get the total match count and the paginated IDs in one query.
const rows = await chQuery<{ profile_id: string; total_count: number }>(`
SELECT
gm.profile_id,
count() OVER () AS total_count
FROM (
SELECT profile_id, max(created_at) AS last_seen
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(groupId)})
AND profile_id != device_id
GROUP BY profile_id
) gm
INNER JOIN (
SELECT id FROM ${TABLE_NAMES.profiles} FINAL
WHERE project_id = ${sqlstring.escape(projectId)}
${searchCondition}
) p ON p.id = gm.profile_id
ORDER BY gm.last_seen DESC
LIMIT ${take}
OFFSET ${offset}
`);
const count = rows[0]?.total_count ?? 0;
const profileIds = rows.map((r) => r.profile_id);
if (profileIds.length === 0) {
return { data: [], count };
}
const profiles = await getProfiles(profileIds, projectId);
const byId = new Map(profiles.map((p) => [p.id, p]));
const data = profileIds
.map((id) => byId.get(id))
.filter(Boolean) as IServiceProfile[];
return { data, count };
}

2
packages/db/src/services/import.service.ts
View File

@@ -355,6 +355,7 @@ export async function createSessionsStartEndEvents(
profile_id: session.profile_id,
project_id: session.project_id,
session_id: session.session_id,
groups: [],
path: firstPath,
origin: firstOrigin,
referrer: firstReferrer,
@@ -390,6 +391,7 @@ export async function createSessionsStartEndEvents(
profile_id: session.profile_id,
project_id: session.project_id,
session_id: session.session_id,
groups: [],
path: lastPath,
origin: lastOrigin,
referrer: firstReferrer,

9
packages/db/src/services/profile.service.ts
View File

@@ -165,7 +165,8 @@ export async function getProfiles(ids: string[], projectId: string) {
any(nullIf(avatar, '')) as avatar,
last_value(is_external) as is_external,
any(properties) as properties,
any(created_at) as created_at
any(created_at) as created_at,
any(groups) as groups
FROM ${TABLE_NAMES.profiles}
WHERE
project_id = ${sqlstring.escape(projectId)} AND
@@ -232,6 +233,7 @@ export interface IServiceProfile {
createdAt: Date;
isExternal: boolean;
projectId: string;
groups: string[];
properties: Record<string, unknown> & {
region?: string;
country?: string;
@@ -259,6 +261,7 @@ export interface IClickhouseProfile {
project_id: string;
is_external: boolean;
created_at: string;
groups: string[];
}
export interface IServiceUpsertProfile {
@@ -270,6 +273,7 @@ export interface IServiceUpsertProfile {
avatar?: string;
properties?: Record<string, unknown>;
isExternal: boolean;
groups?: string[];
}
export function transformProfile({
@@ -288,6 +292,7 @@ export function transformProfile({
id: profile.id,
email: profile.email,
avatar: profile.avatar,
groups: profile.groups ?? [],
};
}
@@ -301,6 +306,7 @@ export function upsertProfile(
properties,
projectId,
isExternal,
groups,
}: IServiceUpsertProfile,
isFromEvent = false
) {
@@ -314,6 +320,7 @@ export function upsertProfile(
project_id: projectId,
created_at: formatClickhouseDate(new Date()),
is_external: isExternal,
groups: groups ?? [],
};
return profileBuffer.add(profile, isFromEvent);

5
packages/db/src/services/session.service.ts
View File

@@ -55,6 +55,7 @@ export interface IClickhouseSession {
version: number;
// Dynamically added
has_replay?: boolean;
groups: string[];
}
export interface IServiceSession {
@@ -95,6 +96,7 @@ export interface IServiceSession {
revenue: number;
profile?: IServiceProfile;
hasReplay?: boolean;
groups: string[];
}
export interface GetSessionListOptions {
@@ -152,6 +154,7 @@ export function transformSession(session: IClickhouseSession): IServiceSession {
revenue: session.revenue,
profile: undefined,
hasReplay: session.has_replay,
groups: session.groups,
};
}
@@ -244,6 +247,7 @@ export async function getSessionList(options: GetSessionListOptions) {
'screen_view_count',
'event_count',
'revenue',
'groups',
];
columns.forEach((column) => {
@@ -292,6 +296,7 @@ export async function getSessionList(options: GetSessionListOptions) {
projectId,
isExternal: false,
properties: {},
groups: [],
},
}));

1
packages/importer/src/providers/mixpanel.test.ts
View File

@@ -314,6 +314,7 @@ describe('mixpanel', () => {
sdk_name: 'mixpanel (web)',
sdk_version: '1.0.0',
session_id: '',
groups: [],
});
});
});

2
packages/importer/src/providers/mixpanel.ts
View File

@@ -396,6 +396,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {
properties,
created_at: createdAt,
is_external: true,
groups: [],
};
}
@@ -536,6 +537,7 @@ export class MixpanelProvider extends BaseImportProvider<MixpanelRawEvent> {
? `${this.provider} (${props.mp_lib})`
: this.provider,
sdk_version: this.version,
groups: [],
};
// TODO: Remove this

1
packages/importer/src/providers/umami.ts
View File

@@ -337,6 +337,7 @@ export class UmamiProvider extends BaseImportProvider<UmamiRawEvent> {
imported_at: new Date().toISOString(),
sdk_name: this.provider,
sdk_version: this.version,
groups: [],
};
}

14
packages/logger/index.ts
View File

@@ -66,7 +66,9 @@ export function createLogger({ name }: { name: string }): ILogger {
const redactSensitiveInfo = winston.format((info) => {
const redactObject = (obj: any): any => {
if (!obj || typeof obj !== 'object') return obj;
if (!obj || typeof obj !== 'object') {
return obj;
}
return Object.keys(obj).reduce((acc, key) => {
const lowerKey = key.toLowerCase();
@@ -85,7 +87,7 @@ export function createLogger({ name }: { name: string }): ILogger {
}, {} as any);
};
return Object.assign({}, info, redactObject(info));
return { ...info, ...redactObject(info) };
});
const transports: winston.transport[] = [];
@@ -96,12 +98,12 @@ export function createLogger({ name }: { name: string }): ILogger {
HyperDX.getWinstonTransport(logLevel, {
detectResources: true,
service,
}),
})
);
format = winston.format.combine(
errorFormatter(),
redactSensitiveInfo(),
winston.format.json(),
winston.format.json()
);
} else {
transports.push(new winston.transports.Console());
@@ -116,7 +118,7 @@ export function createLogger({ name }: { name: string }): ILogger {
const metaStr =
Object.keys(meta).length > 0 ? ` ${JSON.stringify(meta)}` : '';
return `${level} ${message}${metaStr}`;
}),
})
);
}
@@ -126,7 +128,7 @@ export function createLogger({ name }: { name: string }): ILogger {
format,
transports,
silent,
levels: Object.assign({}, customLevels, winston.config.syslog.levels),
levels: { ...customLevels, ...winston.config.syslog.levels },
});
return logger;

5
packages/queue/src/queues.ts
View File

@@ -134,6 +134,10 @@ export type CronQueuePayloadGscSync = {
type: 'gscSync';
payload: undefined;
};
export type CronQueuePayloadFlushGroups = {
type: 'flushGroups';
payload: undefined;
};
export type CronQueuePayload =
| CronQueuePayloadSalt
| CronQueuePayloadFlushEvents
@@ -141,6 +145,7 @@ export type CronQueuePayload =
| CronQueuePayloadFlushProfiles
| CronQueuePayloadFlushProfileBackfill
| CronQueuePayloadFlushReplay
| CronQueuePayloadFlushGroups
| CronQueuePayloadPing
| CronQueuePayloadProject
| CronQueuePayloadInsightsDaily

98
packages/sdks/sdk/src/index.ts
View File

@@ -2,7 +2,9 @@
import type {
IAliasPayload as AliasPayload,
IAssignGroupPayload as AssignGroupPayload,
IDecrementPayload as DecrementPayload,
IGroupPayload as GroupPayload,
IIdentifyPayload as IdentifyPayload,
IIncrementPayload as IncrementPayload,
ITrackHandlerPayload as TrackHandlerPayload,
@@ -12,7 +14,9 @@ import { Api } from './api';
export type {
AliasPayload,
AssignGroupPayload,
DecrementPayload,
GroupPayload,
IdentifyPayload,
IncrementPayload,
TrackHandlerPayload,
@@ -22,8 +26,11 @@ export type {
export interface TrackProperties {
[key: string]: unknown;
profileId?: string;
groups?: string[];
}
export type UpsertGroupPayload = GroupPayload;
export interface OpenPanelOptions {
clientId: string;
clientSecret?: string;
@@ -45,6 +52,7 @@ export class OpenPanel {
api: Api;
options: OpenPanelOptions;
profileId?: string;
groups: string[] = [];
deviceId?: string;
sessionId?: string;
global?: Record<string, unknown>;
@@ -142,14 +150,19 @@ export class OpenPanel {
track(name: string, properties?: TrackProperties) {
this.log('track event', name, properties);
const { groups: groupsOverride, profileId, ...rest } = properties ?? {};
const mergedGroups = [
...new Set([...this.groups, ...(groupsOverride ?? [])]),
];
return this.send({
type: 'track',
payload: {
name,
profileId: properties?.profileId ?? this.profileId,
profileId: profileId ?? this.profileId,
groups: mergedGroups.length > 0 ? mergedGroups : undefined,
properties: {
...(this.global ?? {}),
...(properties ?? {}),
...rest,
},
},
});
@@ -176,6 +189,40 @@ export class OpenPanel {
}
}
upsertGroup(payload: UpsertGroupPayload) {
this.log('upsert group', payload);
return this.send({
type: 'group',
payload,
});
}
setGroup(groupId: string) {
this.log('set group', groupId);
if (!this.groups.includes(groupId)) {
this.groups = [...this.groups, groupId];
}
return this.send({
type: 'assign_group',
payload: {
groupIds: [groupId],
profileId: this.profileId,
},
});
}
setGroups(groupIds: string[]) {
this.log('set groups', groupIds);
this.groups = [...new Set([...this.groups, ...groupIds])];
return this.send({
type: 'assign_group',
payload: {
groupIds,
profileId: this.profileId,
},
});
}
/**
* @deprecated This method is deprecated and will be removed in a future version.
*/
@@ -227,10 +274,46 @@ export class OpenPanel {
clear() {
this.profileId = undefined;
this.groups = [];
this.deviceId = undefined;
this.sessionId = undefined;
}
private buildFlushPayload(
item: TrackHandlerPayload
): TrackHandlerPayload['payload'] {
if (item.type === 'replay') {
return item.payload;
}
if (item.type === 'track') {
const queuedGroups =
'groups' in item.payload ? (item.payload.groups ?? []) : [];
const mergedGroups = [...new Set([...this.groups, ...queuedGroups])];
return {
...item.payload,
profileId: item.payload.profileId ?? this.profileId,
groups: mergedGroups.length > 0 ? mergedGroups : undefined,
};
}
if (
item.type === 'identify' ||
item.type === 'increment' ||
item.type === 'decrement'
) {
return {
...item.payload,
profileId: item.payload.profileId ?? this.profileId,
} as TrackHandlerPayload['payload'];
}
if (item.type === 'assign_group') {
return {
...item.payload,
profileId: item.payload.profileId ?? this.profileId,
};
}
return item.payload;
}
flush() {
const remaining: TrackHandlerPayload[] = [];
for (const item of this.queue) {
@@ -238,16 +321,7 @@ export class OpenPanel {
remaining.push(item);
continue;
}
const payload =
item.type === 'replay'
? item.payload
: {
...item.payload,
profileId:
'profileId' in item.payload
? (item.payload.profileId ?? this.profileId)
: this.profileId,
};
const payload = this.buildFlushPayload(item);
this.send({ ...item, payload } as TrackHandlerPayload);
}
this.queue = remaining;

4
packages/trpc/src/root.ts
View File

@@ -1,11 +1,12 @@
import { authRouter } from './routers/auth';
import { gscRouter } from './routers/gsc';
import { chartRouter } from './routers/chart';
import { chatRouter } from './routers/chat';
import { clientRouter } from './routers/client';
import { dashboardRouter } from './routers/dashboard';
import { emailRouter } from './routers/email';
import { eventRouter } from './routers/event';
import { groupRouter } from './routers/group';
import { gscRouter } from './routers/gsc';
import { importRouter } from './routers/import';
import { insightRouter } from './routers/insight';
import { integrationRouter } from './routers/integration';
@@ -55,6 +56,7 @@ export const appRouter = createTRPCRouter({
widget: widgetRouter,
email: emailRouter,
gsc: gscRouter,
group: groupRouter,
});
// export type definition of API

114
packages/trpc/src/routers/chart.ts
View File

@@ -13,11 +13,12 @@ import {
getChartStartEndDate,
getEventFiltersWhereClause,
getEventMetasCached,
getGroupPropertySelect,
getProfilePropertySelect,
getProfilesCached,
getReportById,
getSelectPropertyKey,
getSettingsForProject,
type IClickhouseProfile,
type IServiceProfile,
onlyReportEvents,
sankeyService,
@@ -354,6 +355,33 @@ export const chartRouter = createTRPCRouter({
const res = await query.execute();
values.push(...res.map((e) => e.property_value));
} else if (property.startsWith('profile.')) {
const selectExpr = getProfilePropertySelect(property);
const query = clix(ch)
.select<{ values: string }>([`distinct ${selectExpr} as values`])
.from(TABLE_NAMES.profiles, true)
.where('project_id', '=', projectId)
.where(selectExpr, '!=', '')
.where(selectExpr, 'IS NOT NULL', null)
.orderBy('created_at', 'DESC')
.limit(100_000);
const res = await query.execute();
values.push(...res.map((r) => String(r.values)).filter(Boolean));
} else if (property.startsWith('group.')) {
const selectExpr = getGroupPropertySelect(property);
const query = clix(ch)
.select<{ values: string }>([`distinct ${selectExpr} as values`])
.from(TABLE_NAMES.groups, true)
.where('project_id', '=', projectId)
.where('deleted', '=', 0)
.where(selectExpr, '!=', '')
.where(selectExpr, 'IS NOT NULL', null)
.orderBy('created_at', 'DESC')
.limit(100_000);
const res = await query.execute();
values.push(...res.map((r) => String(r.values)).filter(Boolean));
} else {
const query = clix(ch)
.select<{ values: string[] }>([
@@ -369,17 +397,6 @@ export const chartRouter = createTRPCRouter({
query.where('name', '=', event);
}
if (property.startsWith('profile.')) {
query.leftAnyJoin(
clix(ch)
.select<IClickhouseProfile>([])
.from(TABLE_NAMES.profiles)
.where('project_id', '=', projectId),
'profile.id = profile_id',
'profile'
);
}
const events = await query.execute();
values.push(
@@ -785,7 +802,7 @@ export const chartRouter = createTRPCRouter({
const { sb, getSql } = createSqlBuilder();
sb.select.profile_id = 'DISTINCT profile_id';
sb.where = getEventFiltersWhereClause(serie.filters);
sb.where = getEventFiltersWhereClause(serie.filters, projectId);
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
sb.where.dateRange = `${clix.toStartOf('created_at', input.interval)} = ${clix.toDate(sqlstring.escape(formatClickhouseDate(dateObj)), input.interval)}`;
if (serie.name !== '*') {
@@ -812,10 +829,22 @@ export const chartRouter = createTRPCRouter({
sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
// Check for group filters/breakdowns and add ARRAY JOIN if needed
const anyFilterOnGroup = serie.filters.some((f) =>
f.name.startsWith('group.')
);
const anyBreakdownOnGroup = input.breakdowns
? Object.keys(input.breakdowns).some((key) => key.startsWith('group.'))
: false;
if (anyFilterOnGroup || anyBreakdownOnGroup) {
sb.joins.groups = 'ARRAY JOIN groups AS _group_id';
sb.joins.groups_cte = `LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`;
}
if (input.breakdowns) {
Object.entries(input.breakdowns).forEach(([key, value]) => {
// Transform property keys (e.g., properties.method -> properties['method'])
const propertyKey = getSelectPropertyKey(key);
const propertyKey = getSelectPropertyKey(key, projectId);
sb.where[`breakdown_${key}`] =
`${propertyKey} = ${sqlstring.escape(value)}`;
});
@@ -858,6 +887,7 @@ export const chartRouter = createTRPCRouter({
funnelWindow: z.number().optional(),
funnelGroup: z.string().optional(),
breakdowns: z.array(z.object({ name: z.string() })).optional(),
breakdownValues: z.array(z.string()).optional(),
range: zRange,
})
)
@@ -870,6 +900,8 @@ export const chartRouter = createTRPCRouter({
showDropoffs = false,
funnelWindow,
funnelGroup,
breakdowns = [],
breakdownValues = [],
} = input;
const { startDate, endDate } = getChartStartEndDate(input, timezone);
@@ -889,9 +921,21 @@ export const chartRouter = createTRPCRouter({
// Get the grouping strategy (profile_id or session_id)
const group = funnelService.getFunnelGroup(funnelGroup);
const anyFilterOnGroup = (eventSeries as IChartEvent[]).some((e) =>
e.filters?.some((f) => f.name.startsWith('group.'))
);
const anyBreakdownOnGroup = breakdowns.some((b) =>
b.name.startsWith('group.')
);
const needsGroupArrayJoin = anyFilterOnGroup || anyBreakdownOnGroup;
// Breakdown selects/groupBy so we can filter by specific breakdown values
const breakdownSelects = breakdowns.map(
(b, index) => `${getSelectPropertyKey(b.name, projectId)} as b_${index}`
);
const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);
// Create funnel CTE using funnel service
// Note: buildFunnelCte always computes windowFunnel per session_id and extracts
// profile_id via argMax to handle identity changes mid-session correctly.
const funnelCte = funnelService.buildFunnelCte({
projectId,
startDate,
@@ -899,8 +943,8 @@ export const chartRouter = createTRPCRouter({
eventSeries: eventSeries as IChartEvent[],
funnelWindowMilliseconds,
timezone,
// No need to add profile_id to additionalSelects/additionalGroupBy
// since buildFunnelCte already extracts it via argMax(profile_id, created_at)
additionalSelects: breakdownSelects,
additionalGroupBy: breakdownGroupBy,
});
// Check for profile filters and add profile join if needed
@@ -917,36 +961,50 @@ export const chartRouter = createTRPCRouter({
);
}
if (needsGroupArrayJoin) {
funnelCte.rawJoin('ARRAY JOIN groups AS _group_id');
funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');
}
// Build main query
const query = clix(ch, timezone);
if (needsGroupArrayJoin) {
query.with(
'_g',
`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`
);
}
query.with('session_funnel', funnelCte);
if (group === 'profile_id') {
// For profile grouping: re-aggregate by profile_id, taking MAX level per profile.
// This ensures a user who completed the funnel with identity change is counted correctly.
// NOTE: Wrap in subquery to avoid ClickHouse resolving `level` in WHERE to the
// `max(level) AS level` alias (ILLEGAL_AGGREGATION error).
const breakdownAggregates =
breakdowns.length > 0
? `, ${breakdowns.map((_, index) => `any(b_${index}) AS b_${index}`).join(', ')}`
: '';
query.with(
'funnel',
'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id'
`SELECT profile_id, max(level) AS level${breakdownAggregates} FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id`
);
} else {
// For session grouping: filter out level = 0 inside the CTE
query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');
}
// Get distinct profile IDs
// NOTE: level != 0 is already filtered inside the funnel CTE above
query.select(['DISTINCT profile_id']).from('funnel');
if (showDropoffs) {
// Show users who dropped off at this step (completed this step but not the next)
query.where('level', '=', targetLevel);
} else {
// Show users who completed at least this step
query.where('level', '>=', targetLevel);
}
// Filter by specific breakdown values when a breakdown row was clicked
breakdowns.forEach((_, index) => {
const value = breakdownValues[index];
if (value !== undefined) {
query.where(`b_${index}`, '=', value);
}
});
// Cap the number of profiles to avoid exceeding ClickHouse max_query_size
// when passing IDs to the next query
query.limit(1000);

1
packages/trpc/src/routers/event.ts
View File

@@ -122,6 +122,7 @@ export const eventRouter = createTRPCRouter({
projectId: z.string(),
profileId: z.string().optional(),
sessionId: z.string().optional(),
groupId: z.string().optional(),
cursor: z.string().optional(),
filters: z.array(zChartEventFilter).default([]),
startDate: z.date().optional(),

233
packages/trpc/src/routers/group.ts Normal file
View File

@@ -0,0 +1,233 @@
import {
chQuery,
createGroup,
deleteGroup,
getGroupById,
getGroupList,
getGroupListCount,
getGroupMemberProfiles,
getGroupPropertyKeys,
getGroupStats,
getGroupsByIds,
getGroupTypes,
TABLE_NAMES,
toNullIfDefaultMinDate,
updateGroup,
} from '@openpanel/db';
import { zCreateGroup, zUpdateGroup } from '@openpanel/validation';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../trpc';
export const groupRouter = createTRPCRouter({
list: protectedProcedure
.input(
z.object({
projectId: z.string(),
cursor: z.number().optional(),
take: z.number().default(50),
search: z.string().optional(),
type: z.string().optional(),
})
)
.query(async ({ input }) => {
const [data, count] = await Promise.all([
getGroupList(input),
getGroupListCount(input),
]);
const stats = await getGroupStats(
input.projectId,
data.map((g) => g.id)
);
return {
data: data.map((g) => ({
...g,
memberCount: stats.get(g.id)?.memberCount ?? 0,
lastActiveAt: stats.get(g.id)?.lastActiveAt ?? null,
})),
meta: { count, take: input.take },
};
}),
byId: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.query(({ input: { id, projectId } }) => {
return getGroupById(id, projectId);
}),
create: protectedProcedure
.input(zCreateGroup)
.mutation(({ input }) => {
return createGroup(input);
}),
update: protectedProcedure
.input(zUpdateGroup)
.mutation(({ input: { id, projectId, ...data } }) => {
return updateGroup(id, projectId, data);
}),
delete: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.mutation(({ input: { id, projectId } }) => {
return deleteGroup(id, projectId);
}),
types: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(({ input: { projectId } }) => {
return getGroupTypes(projectId);
}),
metrics: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.query(async ({ input: { id, projectId } }) => {
const data = await chQuery<{
totalEvents: number;
uniqueProfiles: number;
firstSeen: string;
lastSeen: string;
}>(`
SELECT
count() AS totalEvents,
uniqExact(profile_id) AS uniqueProfiles,
min(created_at) AS firstSeen,
max(created_at) AS lastSeen
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(id)})
`);
return {
totalEvents: data[0]?.totalEvents ?? 0,
uniqueProfiles: data[0]?.uniqueProfiles ?? 0,
firstSeen: toNullIfDefaultMinDate(data[0]?.firstSeen),
lastSeen: toNullIfDefaultMinDate(data[0]?.lastSeen),
};
}),
activity: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.query(({ input: { id, projectId } }) => {
return chQuery<{ count: number; date: string }>(`
SELECT count() AS count, toStartOfDay(created_at) AS date
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(id)})
GROUP BY date
ORDER BY date DESC
`);
}),
members: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.query(({ input: { id, projectId } }) => {
return chQuery<{
profileId: string;
lastSeen: string;
eventCount: number;
}>(`
SELECT
profile_id AS profileId,
max(created_at) AS lastSeen,
count() AS eventCount
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(id)})
AND profile_id != device_id
GROUP BY profile_id
ORDER BY lastSeen DESC, eventCount DESC
LIMIT 50
`);
}),
listProfiles: protectedProcedure
.input(
z.object({
projectId: z.string(),
groupId: z.string(),
cursor: z.number().optional(),
take: z.number().default(50),
search: z.string().optional(),
})
)
.query(async ({ input }) => {
const { data, count } = await getGroupMemberProfiles({
projectId: input.projectId,
groupId: input.groupId,
cursor: input.cursor,
take: input.take,
search: input.search,
});
return {
data,
meta: { count, pageCount: input.take },
};
}),
mostEvents: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.query(({ input: { id, projectId } }) => {
return chQuery<{ count: number; name: string }>(`
SELECT count() as count, name
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(id)})
AND name NOT IN ('screen_view', 'session_start', 'session_end')
GROUP BY name
ORDER BY count DESC
LIMIT 10
`);
}),
popularRoutes: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.query(({ input: { id, projectId } }) => {
return chQuery<{ count: number; path: string }>(`
SELECT count() as count, path
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(id)})
AND name = 'screen_view'
GROUP BY path
ORDER BY count DESC
LIMIT 10
`);
}),
memberGrowth: protectedProcedure
.input(z.object({ id: z.string(), projectId: z.string() }))
.query(({ input: { id, projectId } }) => {
return chQuery<{ date: string; count: number }>(`
SELECT
toDate(toStartOfDay(min_date)) AS date,
count() AS count
FROM (
SELECT profile_id, min(created_at) AS min_date
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(projectId)}
AND has(groups, ${sqlstring.escape(id)})
AND profile_id != device_id
AND created_at >= now() - INTERVAL 30 DAY
GROUP BY profile_id
)
GROUP BY date
ORDER BY date ASC WITH FILL
FROM toDate(now() - INTERVAL 29 DAY)
TO toDate(now() + INTERVAL 1 DAY)
STEP 1
`);
}),
properties: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(({ input: { projectId } }) => {
return getGroupPropertyKeys(projectId);
}),
listByIds: protectedProcedure
.input(z.object({ projectId: z.string(), ids: z.array(z.string()) }))
.query(({ input: { projectId, ids } }) => {
return getGroupsByIds(projectId, ids);
}),
});

26
packages/validation/src/index.ts
View File

@@ -540,6 +540,32 @@ export const zCheckout = z.object({
});
export type ICheckout = z.infer<typeof zCheckout>;
export const zGroupId = z
.string()
.min(1)
.regex(
/^[a-z0-9_-]+$/,
'ID must only contain lowercase letters, digits, hyphens, or underscores',
);
export const zCreateGroup = z.object({
id: zGroupId,
projectId: z.string(),
type: z.string().min(1),
name: z.string().min(1),
properties: z.record(z.string()).default({}),
});
export type ICreateGroup = z.infer<typeof zCreateGroup>;
export const zUpdateGroup = z.object({
id: z.string().min(1),
projectId: z.string(),
type: z.string().min(1).optional(),
name: z.string().min(1).optional(),
properties: z.record(z.string()).optional(),
});
export type IUpdateGroup = z.infer<typeof zUpdateGroup>;
export const zEditOrganization = z.object({
id: z.string().min(2),
name: z.string().min(2),

25
packages/validation/src/track.validation.ts
View File

@@ -2,11 +2,24 @@ import { RESERVED_EVENT_NAMES } from '@openpanel/constants';
import { z } from 'zod';
import { isBlockedEventName } from './event-blocklist';
export const zGroupPayload = z.object({
id: z.string().min(1),
type: z.string().min(1),
name: z.string().min(1),
properties: z.record(z.unknown()).optional(),
});
export const zAssignGroupPayload = z.object({
groupIds: z.array(z.string().min(1)),
profileId: z.union([z.string().min(1), z.number()]).optional(),
});
export const zTrackPayload = z
.object({
name: z.string().min(1),
properties: z.record(z.string(), z.unknown()).optional(),
profileId: z.string().or(z.number()).optional(),
profileId: z.union([z.string().min(1), z.number()]).optional(),
groups: z.array(z.string().min(1)).optional(),
})
.refine((data) => !RESERVED_EVENT_NAMES.includes(data.name as any), {
message: `Event name cannot be one of the reserved names: ${RESERVED_EVENT_NAMES.join(', ')}`,
@@ -97,6 +110,14 @@ export const zTrackHandlerPayload = z.discriminatedUnion('type', [
type: z.literal('replay'),
payload: zReplayPayload,
}),
z.object({
type: z.literal('group'),
payload: zGroupPayload,
}),
z.object({
type: z.literal('assign_group'),
payload: zAssignGroupPayload,
}),
]);
export type ITrackPayload = z.infer<typeof zTrackPayload>;
@@ -105,6 +126,8 @@ export type IIncrementPayload = z.infer<typeof zIncrementPayload>;
export type IDecrementPayload = z.infer<typeof zDecrementPayload>;
export type IAliasPayload = z.infer<typeof zAliasPayload>;
export type IReplayPayload = z.infer<typeof zReplayPayload>;
export type IGroupPayload = z.infer<typeof zGroupPayload>;
export type IAssignGroupPayload = z.infer<typeof zAssignGroupPayload>;
export type ITrackHandlerPayload = z.infer<typeof zTrackHandlerPayload>;
// Deprecated types for beta version of the SDKs