feature(dashboard): refactor overview
fix(lint)
This commit is contained in:
Carl-Gerhard Lindesvärd
committed by
Carl-Gerhard Lindesvärd
Carl-Gerhard Lindesvärd
parent
b035c0d586
commit
a1eb4a296f
@@ -1,7 +1,9 @@
|
||||
import { BotBuffer as BotBufferRedis } from './bot-buffer-redis';
|
||||
import { EventBuffer as EventBufferRedis } from './event-buffer-redis';
|
||||
import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer-redis';
|
||||
import { SessionBuffer } from './session-buffer';
|
||||
|
||||
export const eventBuffer = new EventBufferRedis();
|
||||
export const profileBuffer = new ProfileBufferRedis();
|
||||
export const botBuffer = new BotBufferRedis();
|
||||
export const sessionBuffer = new SessionBuffer();
|
||||
|
||||
211
packages/db/src/buffers/session-buffer.ts
Normal file
211
packages/db/src/buffers/session-buffer.ts
Normal file
@@ -0,0 +1,211 @@
|
||||
import { type Redis, getRedisCache, runEvery } from '@openpanel/redis';
|
||||
|
||||
import { toDots } from '@openpanel/common';
|
||||
import { getSafeJson } from '@openpanel/json';
|
||||
import { assocPath, clone } from 'ramda';
|
||||
import { TABLE_NAMES, ch } from '../clickhouse/client';
|
||||
import type { IClickhouseEvent } from '../services/event.service';
|
||||
import type { IClickhouseSession } from '../services/session.service';
|
||||
import { BaseBuffer } from './base-buffer';
|
||||
|
||||
export class SessionBuffer extends BaseBuffer {
|
||||
private batchSize = process.env.BOT_BUFFER_BATCH_SIZE
|
||||
? Number.parseInt(process.env.BOT_BUFFER_BATCH_SIZE, 10)
|
||||
: 2;
|
||||
|
||||
private readonly redisKey = 'session-buffer';
|
||||
private redis: Redis;
|
||||
constructor() {
|
||||
super({
|
||||
name: 'session',
|
||||
onFlush: async () => {
|
||||
await this.processBuffer();
|
||||
},
|
||||
});
|
||||
this.redis = getRedisCache();
|
||||
}
|
||||
|
||||
async getExistingSession(sessionId: string) {
|
||||
const hit = await this.redis.get(`session:${sessionId}`);
|
||||
|
||||
if (hit) {
|
||||
return getSafeJson<IClickhouseSession>(hit);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async getSession(
|
||||
event: IClickhouseEvent,
|
||||
): Promise<[IClickhouseSession] | [IClickhouseSession, IClickhouseSession]> {
|
||||
const existingSession = await this.getExistingSession(event.session_id);
|
||||
|
||||
if (existingSession) {
|
||||
const oldSession = assocPath(['sign'], -1, clone(existingSession));
|
||||
const newSession = assocPath(['sign'], 1, clone(existingSession));
|
||||
|
||||
newSession.ended_at = event.created_at;
|
||||
newSession.version = existingSession.version + 1;
|
||||
if (!newSession.entry_path) {
|
||||
newSession.entry_path = event.path;
|
||||
newSession.entry_origin = event.origin;
|
||||
}
|
||||
newSession.exit_path = event.path;
|
||||
newSession.exit_origin = event.origin;
|
||||
newSession.duration =
|
||||
new Date(newSession.ended_at).getTime() -
|
||||
new Date(newSession.created_at).getTime();
|
||||
newSession.properties = toDots({
|
||||
...(event.properties || {}),
|
||||
...(newSession.properties || {}),
|
||||
});
|
||||
// newSession.revenue += event.properties?.__revenue ?? 0;
|
||||
|
||||
if (event.name === 'screen_view') {
|
||||
newSession.screen_views.push(event.path);
|
||||
newSession.screen_view_count += 1;
|
||||
} else {
|
||||
newSession.event_count += 1;
|
||||
}
|
||||
|
||||
if (newSession.screen_view_count > 1) {
|
||||
newSession.is_bounce = false;
|
||||
}
|
||||
|
||||
// If the profile_id is set and it's different from the device_id, we need to update the profile_id
|
||||
if (event.profile_id && event.profile_id !== event.device_id) {
|
||||
newSession.profile_id = event.profile_id;
|
||||
}
|
||||
|
||||
return [newSession, oldSession];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: event.session_id,
|
||||
is_bounce: true,
|
||||
profile_id: event.profile_id,
|
||||
project_id: event.project_id,
|
||||
device_id: event.device_id,
|
||||
created_at: event.created_at,
|
||||
ended_at: event.created_at,
|
||||
event_count: event.name === 'screen_view' ? 0 : 1,
|
||||
screen_view_count: event.name === 'screen_view' ? 1 : 0,
|
||||
screen_views: event.name === 'screen_view' ? [event.path] : [],
|
||||
entry_path: event.path,
|
||||
entry_origin: event.origin,
|
||||
exit_path: event.path,
|
||||
exit_origin: event.origin,
|
||||
revenue: 0,
|
||||
referrer: event.referrer,
|
||||
referrer_name: event.referrer_name,
|
||||
referrer_type: event.referrer_type,
|
||||
os: event.os,
|
||||
os_version: event.os_version,
|
||||
browser: event.browser,
|
||||
browser_version: event.browser_version,
|
||||
device: event.device,
|
||||
brand: event.brand,
|
||||
model: event.model,
|
||||
country: event.country,
|
||||
region: event.region,
|
||||
city: event.city,
|
||||
longitude: event.longitude ?? null,
|
||||
latitude: event.latitude ?? null,
|
||||
duration: event.duration,
|
||||
utm_medium: event.properties?.['__query.utm_medium']
|
||||
? String(event.properties?.['__query.utm_medium'])
|
||||
: '',
|
||||
utm_source: event.properties?.['__query.utm_source']
|
||||
? String(event.properties?.['__query.utm_source'])
|
||||
: '',
|
||||
utm_campaign: event.properties?.['__query.utm_campaign']
|
||||
? String(event.properties?.['__query.utm_campaign'])
|
||||
: '',
|
||||
utm_content: event.properties?.['__query.utm_content']
|
||||
? String(event.properties?.['__query.utm_content'])
|
||||
: '',
|
||||
utm_term: event.properties?.['__query.utm_term']
|
||||
? String(event.properties?.['__query.utm_term'])
|
||||
: '',
|
||||
sign: 1,
|
||||
version: 1,
|
||||
properties: toDots(event.properties || {}),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
async add(event: IClickhouseEvent) {
|
||||
if (!event.session_id) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (['session_start', 'session_end'].includes(event.name)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Plural since we will delete the old session with sign column
|
||||
const sessions = await this.getSession(event);
|
||||
const [newSession] = sessions;
|
||||
|
||||
console.log(`Adding sessions ${sessions.length}`);
|
||||
|
||||
const multi = this.redis.multi();
|
||||
multi.set(
|
||||
`session:${newSession.id}`,
|
||||
JSON.stringify(newSession),
|
||||
'EX',
|
||||
60 * 60,
|
||||
);
|
||||
for (const session of sessions) {
|
||||
multi.rpush(this.redisKey, JSON.stringify(session));
|
||||
}
|
||||
await multi.exec();
|
||||
|
||||
// Check buffer length
|
||||
const bufferLength = await this.redis.llen(this.redisKey);
|
||||
|
||||
if (bufferLength >= this.batchSize) {
|
||||
await this.tryFlush();
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to add bot event', { error });
|
||||
}
|
||||
}
|
||||
|
||||
async processBuffer() {
|
||||
try {
|
||||
// Get events from the start without removing them
|
||||
const events = await this.redis.lrange(
|
||||
this.redisKey,
|
||||
0,
|
||||
this.batchSize - 1,
|
||||
);
|
||||
|
||||
if (events.length === 0) return;
|
||||
|
||||
const sessions = events.map((e) => getSafeJson<IClickhouseSession>(e));
|
||||
|
||||
// Insert to ClickHouse
|
||||
await ch.insert({
|
||||
table: TABLE_NAMES.sessions,
|
||||
values: sessions,
|
||||
format: 'JSONEachRow',
|
||||
});
|
||||
|
||||
// Only remove events after successful insert
|
||||
await this.redis.ltrim(this.redisKey, events.length, -1);
|
||||
|
||||
this.logger.info('Processed sessions', {
|
||||
count: events.length,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to process buffer', { error });
|
||||
}
|
||||
}
|
||||
|
||||
async getBufferSize() {
|
||||
return getRedisCache().llen(this.redisKey);
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ export { createClient };
|
||||
const logger = createLogger({ name: 'clickhouse' });
|
||||
|
||||
import type { Logger } from '@clickhouse/client';
|
||||
import { getTimezoneFromDateString } from '@openpanel/common';
|
||||
|
||||
// All three LogParams types are exported by the client
|
||||
interface LogParams {
|
||||
@@ -55,6 +56,7 @@ export const TABLE_NAMES = {
|
||||
event_names_mv: 'distinct_event_names_mv',
|
||||
event_property_values_mv: 'event_property_values_mv',
|
||||
cohort_events_mv: 'cohort_events_mv',
|
||||
sessions: 'sessions',
|
||||
};
|
||||
|
||||
export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {
|
||||
|
||||
@@ -39,10 +39,10 @@ export const chMigrationClient = createClient({
|
||||
request_timeout: 3600000, // 1 hour in milliseconds
|
||||
keep_alive: {
|
||||
enabled: true,
|
||||
idle_socket_ttl: 8000,
|
||||
},
|
||||
compression: {
|
||||
request: true,
|
||||
response: true,
|
||||
},
|
||||
clickhouse_settings: {
|
||||
wait_end_of_query: 1,
|
||||
|
||||
730
packages/db/src/clickhouse/query-builder.ts
Normal file
730
packages/db/src/clickhouse/query-builder.ts
Normal file
@@ -0,0 +1,730 @@
|
||||
import type { ClickHouseClient, ResponseJSON } from '@clickhouse/client';
|
||||
import type { IInterval } from '@openpanel/validation';
|
||||
import { escape } from 'sqlstring';
|
||||
|
||||
type SqlValue = string | number | boolean | Date | null | Expression;
|
||||
type SqlParam = SqlValue | SqlValue[];
|
||||
type Operator =
|
||||
| '='
|
||||
| '>'
|
||||
| '<'
|
||||
| '>='
|
||||
| '<='
|
||||
| '!='
|
||||
| 'IN'
|
||||
| 'NOT IN'
|
||||
| 'LIKE'
|
||||
| 'NOT LIKE'
|
||||
| 'IS NULL'
|
||||
| 'IS NOT NULL'
|
||||
| 'BETWEEN';
|
||||
|
||||
type CTE = {
|
||||
name: string;
|
||||
query: Query | string;
|
||||
};
|
||||
|
||||
type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS';
|
||||
|
||||
type WhereCondition = {
|
||||
condition: string;
|
||||
operator: 'AND' | 'OR';
|
||||
isGroup?: boolean;
|
||||
};
|
||||
|
||||
type ConditionalCallback = (query: Query) => void;
|
||||
|
||||
class Expression {
|
||||
constructor(private expression: string) {}
|
||||
|
||||
toString() {
|
||||
return this.expression;
|
||||
}
|
||||
}
|
||||
|
||||
export class Query<T = any> {
|
||||
private _select: string[] = [];
|
||||
private _except: string[] = [];
|
||||
private _from?: string | Expression;
|
||||
private _where: WhereCondition[] = [];
|
||||
private _groupBy: string[] = [];
|
||||
private _rollup = false;
|
||||
private _having: { condition: string; operator: 'AND' | 'OR' }[] = [];
|
||||
private _orderBy: {
|
||||
column: string;
|
||||
direction: 'ASC' | 'DESC';
|
||||
}[] = [];
|
||||
private _limit?: number;
|
||||
private _offset?: number;
|
||||
private _final = false;
|
||||
private _settings: Record<string, string> = {};
|
||||
private _ctes: CTE[] = [];
|
||||
private _joins: {
|
||||
type: JoinType;
|
||||
table: string | Expression;
|
||||
condition: string;
|
||||
alias?: string;
|
||||
}[] = [];
|
||||
private _skipNext = false;
|
||||
private _fill?: {
|
||||
from: string | Date;
|
||||
to: string | Date;
|
||||
step: string;
|
||||
};
|
||||
private _transform?: Record<string, (item: T) => any>;
|
||||
private _union?: Query;
|
||||
constructor(private client: ClickHouseClient) {}
|
||||
|
||||
// Select methods
|
||||
select<U>(
|
||||
columns: (string | null | undefined | false)[],
|
||||
type: 'merge' | 'replace' = 'replace',
|
||||
): Query<U> {
|
||||
if (this._skipNext) return this as unknown as Query<U>;
|
||||
if (type === 'merge') {
|
||||
this._select = [
|
||||
...this._select,
|
||||
...columns.filter((col): col is string => Boolean(col)),
|
||||
];
|
||||
} else {
|
||||
this._select = columns.filter((col): col is string => Boolean(col));
|
||||
}
|
||||
return this as unknown as Query<U>;
|
||||
}
|
||||
|
||||
except(columns: string[]): this {
|
||||
this._except = [...this._except, ...columns];
|
||||
return this;
|
||||
}
|
||||
|
||||
rollup(): this {
|
||||
this._rollup = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
// From methods
|
||||
from(table: string | Expression, final = false): this {
|
||||
this._from = table;
|
||||
this._final = final;
|
||||
return this;
|
||||
}
|
||||
|
||||
union(query: Query): this {
|
||||
this._union = query;
|
||||
return this;
|
||||
}
|
||||
|
||||
// Where methods
|
||||
private escapeValue(value: SqlParam): string {
|
||||
if (value === null) return 'NULL';
|
||||
if (value instanceof Expression) return `(${value.toString()})`;
|
||||
if (Array.isArray(value)) {
|
||||
return `(${value.map((v) => this.escapeValue(v)).join(', ')})`;
|
||||
}
|
||||
if (value instanceof Date) {
|
||||
return escape(clix.datetime(value));
|
||||
}
|
||||
return escape(value);
|
||||
}
|
||||
|
||||
where(column: string, operator: Operator, value?: SqlParam): this {
|
||||
if (this._skipNext) return this;
|
||||
const condition = this.buildCondition(column, operator, value);
|
||||
this._where.push({ condition, operator: 'AND' });
|
||||
return this;
|
||||
}
|
||||
|
||||
public buildCondition(
|
||||
column: string,
|
||||
operator: Operator,
|
||||
value?: SqlParam,
|
||||
): string {
|
||||
switch (operator) {
|
||||
case 'IS NULL':
|
||||
return `${column} IS NULL`;
|
||||
case 'IS NOT NULL':
|
||||
return `${column} IS NOT NULL`;
|
||||
case 'BETWEEN':
|
||||
if (Array.isArray(value) && value.length === 2) {
|
||||
return `${column} BETWEEN ${this.escapeValue(value[0]!)} AND ${this.escapeValue(value[1]!)}`;
|
||||
}
|
||||
throw new Error('BETWEEN operator requires an array of two values');
|
||||
case 'IN':
|
||||
case 'NOT IN':
|
||||
if (!Array.isArray(value) && !(value instanceof Expression)) {
|
||||
throw new Error(`${operator} operator requires an array value`);
|
||||
}
|
||||
return `${column} ${operator} (${this.escapeValue(value)})`;
|
||||
default:
|
||||
return `${column} ${operator} ${this.escapeValue(value!)}`;
|
||||
}
|
||||
}
|
||||
|
||||
andWhere(column: string, operator: Operator, value?: SqlParam): this {
|
||||
const condition = this.buildCondition(column, operator, value);
|
||||
this._where.push({ condition, operator: 'AND' });
|
||||
return this;
|
||||
}
|
||||
|
||||
rawWhere(condition: string): this {
|
||||
if (condition) {
|
||||
this._where.push({ condition, operator: 'AND' });
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
orWhere(column: string, operator: Operator, value?: SqlParam): this {
|
||||
const condition = this.buildCondition(column, operator, value);
|
||||
this._where.push({ condition, operator: 'OR' });
|
||||
return this;
|
||||
}
|
||||
|
||||
// Group by methods
|
||||
groupBy(columns: (string | null | undefined | false)[]): this {
|
||||
this._groupBy = columns.filter((col): col is string => Boolean(col));
|
||||
return this;
|
||||
}
|
||||
|
||||
// Having methods
|
||||
having(column: string, operator: Operator, value: SqlParam): this {
|
||||
const condition = this.buildCondition(column, operator, value);
|
||||
this._having.push({ condition, operator: 'AND' });
|
||||
return this;
|
||||
}
|
||||
|
||||
andHaving(column: string, operator: Operator, value: SqlParam): this {
|
||||
const condition = this.buildCondition(column, operator, value);
|
||||
this._having.push({ condition, operator: 'AND' });
|
||||
return this;
|
||||
}
|
||||
|
||||
orHaving(column: string, operator: Operator, value: SqlParam): this {
|
||||
const condition = this.buildCondition(column, operator, value);
|
||||
this._having.push({ condition, operator: 'OR' });
|
||||
return this;
|
||||
}
|
||||
|
||||
// Order by methods
|
||||
orderBy(column: string, direction: 'ASC' | 'DESC' = 'ASC'): this {
|
||||
if (this._skipNext) return this;
|
||||
this._orderBy.push({ column, direction });
|
||||
return this;
|
||||
}
|
||||
|
||||
// Limit and offset
|
||||
limit(limit?: number): this {
|
||||
if (limit !== undefined) {
|
||||
this._limit = limit;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
offset(offset?: number): this {
|
||||
if (offset !== undefined) {
|
||||
this._offset = offset;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
// Settings
|
||||
settings(settings: Record<string, string>): this {
|
||||
Object.assign(this._settings, settings);
|
||||
return this;
|
||||
}
|
||||
|
||||
with(name: string, query: Query | string): this {
|
||||
this._ctes.push({ name, query });
|
||||
return this;
|
||||
}
|
||||
|
||||
// Fill
|
||||
fill(from: string | Date, to: string | Date, step: string): this {
|
||||
this._fill = {
|
||||
from: this.escapeDate(from),
|
||||
to: this.escapeDate(to),
|
||||
step: step,
|
||||
};
|
||||
return this;
|
||||
}
|
||||
|
||||
private escapeDate(value: string | Date): string {
|
||||
if (value instanceof Date) {
|
||||
return clix.datetime(value);
|
||||
}
|
||||
|
||||
return value.replaceAll(/\d{4}-\d{2}-\d{2}([\s\:\d\.]+)?/g, (match) => {
|
||||
return escape(match);
|
||||
});
|
||||
}
|
||||
|
||||
// Add join methods
|
||||
join(table: string | Expression, condition: string, alias?: string): this {
|
||||
return this.joinWithType('INNER', table, condition, alias);
|
||||
}
|
||||
|
||||
innerJoin(
|
||||
table: string | Expression,
|
||||
condition: string,
|
||||
alias?: string,
|
||||
): this {
|
||||
return this.joinWithType('INNER', table, condition, alias);
|
||||
}
|
||||
|
||||
leftJoin(
|
||||
table: string | Expression,
|
||||
condition: string,
|
||||
alias?: string,
|
||||
): this {
|
||||
return this.joinWithType('LEFT', table, condition, alias);
|
||||
}
|
||||
|
||||
rightJoin(
|
||||
table: string | Expression,
|
||||
condition: string,
|
||||
alias?: string,
|
||||
): this {
|
||||
return this.joinWithType('RIGHT', table, condition, alias);
|
||||
}
|
||||
|
||||
fullJoin(
|
||||
table: string | Expression,
|
||||
condition: string,
|
||||
alias?: string,
|
||||
): this {
|
||||
return this.joinWithType('FULL', table, condition, alias);
|
||||
}
|
||||
|
||||
crossJoin(table: string | Expression, alias?: string): this {
|
||||
return this.joinWithType('CROSS', table, '', alias);
|
||||
}
|
||||
|
||||
private joinWithType(
|
||||
type: JoinType,
|
||||
table: string | Expression,
|
||||
condition: string,
|
||||
alias?: string,
|
||||
): this {
|
||||
if (this._skipNext) return this;
|
||||
this._joins.push({
|
||||
type,
|
||||
table,
|
||||
condition: this.escapeDate(condition),
|
||||
alias,
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
// Add methods for grouping conditions
|
||||
whereGroup(): WhereGroupBuilder {
|
||||
return new WhereGroupBuilder(this, 'AND');
|
||||
}
|
||||
|
||||
orWhereGroup(): WhereGroupBuilder {
|
||||
return new WhereGroupBuilder(this, 'OR');
|
||||
}
|
||||
|
||||
// Update buildQuery method's WHERE section
|
||||
private buildWhereConditions(conditions: WhereCondition[]): string {
|
||||
return conditions
|
||||
.map((w, i) => {
|
||||
const condition = w.isGroup ? `(${w.condition})` : w.condition;
|
||||
return i === 0 ? condition : `${w.operator} ${condition}`;
|
||||
})
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
private buildQuery(): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
// Add WITH clause if CTEs exist
|
||||
if (this._ctes.length > 0) {
|
||||
const cteStatements = this._ctes.map((cte) => {
|
||||
const queryStr =
|
||||
typeof cte.query === 'string' ? cte.query : cte.query.toSQL();
|
||||
return `${cte.name} AS (${queryStr})`;
|
||||
});
|
||||
parts.push(`WITH ${cteStatements.join(', ')}`);
|
||||
}
|
||||
|
||||
// SELECT
|
||||
if (this._select.length > 0) {
|
||||
parts.push('SELECT', this._select.map(this.escapeDate).join(', '));
|
||||
} else {
|
||||
parts.push('SELECT *');
|
||||
}
|
||||
|
||||
if (this._except.length > 0) {
|
||||
parts.push('EXCEPT', `(${this._except.map(this.escapeDate).join(', ')})`);
|
||||
}
|
||||
|
||||
// FROM
|
||||
if (this._from) {
|
||||
if (this._from instanceof Expression) {
|
||||
parts.push(`FROM (${this._from.toString()})`);
|
||||
} else {
|
||||
parts.push(`FROM ${this._from}${this._final ? ' FINAL' : ''}`);
|
||||
}
|
||||
|
||||
// Add joins
|
||||
this._joins.forEach((join) => {
|
||||
const aliasClause = join.alias ? ` ${join.alias} ` : ' ';
|
||||
const conditionStr = join.condition ? `ON ${join.condition}` : '';
|
||||
parts.push(
|
||||
`${join.type} JOIN ${join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// WHERE
|
||||
if (this._where.length > 0) {
|
||||
parts.push('WHERE', this.buildWhereConditions(this._where));
|
||||
}
|
||||
|
||||
// GROUP BY
|
||||
if (this._groupBy.length > 0) {
|
||||
parts.push('GROUP BY', this._groupBy.join(', '));
|
||||
}
|
||||
|
||||
if (this._rollup) {
|
||||
parts.push('WITH ROLLUP');
|
||||
}
|
||||
|
||||
// HAVING
|
||||
if (this._having.length > 0) {
|
||||
const conditions = this._having.map((h, i) => {
|
||||
return i === 0 ? h.condition : `${h.operator} ${h.condition}`;
|
||||
});
|
||||
parts.push('HAVING', conditions.join(' '));
|
||||
}
|
||||
|
||||
// ORDER BY
|
||||
if (this._orderBy.length > 0) {
|
||||
const orderBy = this._orderBy.map((o) => {
|
||||
const col = o.column;
|
||||
return `${col} ${o.direction}`;
|
||||
});
|
||||
parts.push('ORDER BY', orderBy.join(', '));
|
||||
}
|
||||
|
||||
// Add FILL clause after ORDER BY
|
||||
if (this._fill) {
|
||||
const fromDate =
|
||||
this._fill.from instanceof Date
|
||||
? clix.datetime(this._fill.from)
|
||||
: this._fill.from;
|
||||
const toDate =
|
||||
this._fill.to instanceof Date
|
||||
? clix.datetime(this._fill.to)
|
||||
: this._fill.to;
|
||||
|
||||
parts.push('WITH FILL');
|
||||
parts.push(`FROM ${fromDate}`);
|
||||
parts.push(`TO ${toDate}`);
|
||||
parts.push(`STEP ${this._fill.step}`);
|
||||
}
|
||||
|
||||
// LIMIT & OFFSET
|
||||
if (this._limit !== undefined) {
|
||||
parts.push(`LIMIT ${this._limit}`);
|
||||
if (this._offset !== undefined) {
|
||||
parts.push(`OFFSET ${this._offset}`);
|
||||
}
|
||||
}
|
||||
|
||||
// SETTINGS
|
||||
if (Object.keys(this._settings).length > 0) {
|
||||
const settings = Object.entries(this._settings)
|
||||
.map(([key, value]) => `${key} = ${value}`)
|
||||
.join(', ');
|
||||
parts.push(`SETTINGS ${settings}`);
|
||||
}
|
||||
|
||||
if (this._union) {
|
||||
parts.push('UNION ALL', this._union.buildQuery());
|
||||
}
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
transformJson<E extends ResponseJSON<any>>(json: E): E {
|
||||
const keys = Object.keys(json.data[0] || {});
|
||||
const response = {
|
||||
...json,
|
||||
data: json.data.map((item) => {
|
||||
return keys.reduce((acc, key) => {
|
||||
const meta = json.meta?.find((m) => m.name === key);
|
||||
const transformer = this._transform?.[key];
|
||||
|
||||
if (transformer) {
|
||||
return {
|
||||
...acc,
|
||||
[key]: transformer(item),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...acc,
|
||||
[key]:
|
||||
item[key] && meta?.type.includes('Int')
|
||||
? Number.parseFloat(item[key] as string)
|
||||
: item[key],
|
||||
};
|
||||
}, {} as T);
|
||||
}),
|
||||
};
|
||||
return response;
|
||||
}
|
||||
|
||||
transform(transformations: Record<string, (item: T) => any>): this {
|
||||
this._transform = transformations;
|
||||
return this;
|
||||
}
|
||||
|
||||
// Execution methods
|
||||
async execute(): Promise<T[]> {
|
||||
const query = this.buildQuery();
|
||||
console.log('TEST QUERY ----->');
|
||||
console.log(query);
|
||||
console.log('<----------');
|
||||
const perf = performance.now();
|
||||
try {
|
||||
const result = await this.client.query({
|
||||
query,
|
||||
});
|
||||
const json = await result.json<T>();
|
||||
const perf2 = performance.now();
|
||||
console.log(`PERF: ${perf2 - perf}ms`);
|
||||
return this.transformJson(json).data;
|
||||
} catch (error) {
|
||||
console.log('ERROR ----->');
|
||||
console.log(error);
|
||||
console.log('<----------');
|
||||
console.log(query);
|
||||
console.log('<----------');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Debug methods
|
||||
toSQL(): string {
|
||||
return this.buildQuery();
|
||||
}
|
||||
|
||||
// Add method to add where conditions (for internal use)
|
||||
_addWhereCondition(condition: WhereCondition): this {
|
||||
this._where.push(condition);
|
||||
return this;
|
||||
}
|
||||
|
||||
if(condition: any): this {
|
||||
this._skipNext = !condition;
|
||||
return this;
|
||||
}
|
||||
|
||||
endIf(): this {
|
||||
this._skipNext = false;
|
||||
return this;
|
||||
}
|
||||
|
||||
// Add method for callback-style conditionals
|
||||
when(condition: boolean, callback?: ConditionalCallback): this {
|
||||
if (condition && callback) {
|
||||
callback(this);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
clone(): Query<T> {
|
||||
return new Query(this.client).merge(this);
|
||||
}
|
||||
|
||||
// Add merge method
|
||||
merge(query: Query): this {
|
||||
if (this._skipNext) return this;
|
||||
|
||||
this._from = query._from;
|
||||
|
||||
this._select = [...this._select, ...query._select];
|
||||
|
||||
this._except = [...this._except, ...query._except];
|
||||
|
||||
// Merge WHERE conditions
|
||||
this._where = [...this._where, ...query._where];
|
||||
|
||||
// Merge CTEs
|
||||
this._ctes = [...this._ctes, ...query._ctes];
|
||||
|
||||
// Merge JOINS
|
||||
this._joins = [...this._joins, ...query._joins];
|
||||
|
||||
// Merge settings
|
||||
this._settings = { ...this._settings, ...query._settings };
|
||||
|
||||
// Take the most restrictive LIMIT
|
||||
if (query._limit !== undefined) {
|
||||
this._limit =
|
||||
this._limit === undefined
|
||||
? query._limit
|
||||
: Math.min(this._limit, query._limit);
|
||||
}
|
||||
|
||||
// Merge ORDER BY
|
||||
this._orderBy = [...this._orderBy, ...query._orderBy];
|
||||
|
||||
// Merge GROUP BY
|
||||
this._groupBy = [...this._groupBy, ...query._groupBy];
|
||||
|
||||
// Merge HAVING conditions
|
||||
this._having = [...this._having, ...query._having];
|
||||
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
// Add this new class for building where groups
|
||||
export class WhereGroupBuilder {
|
||||
private conditions: WhereCondition[] = [];
|
||||
|
||||
constructor(
|
||||
private query: Query,
|
||||
private groupOperator: 'AND' | 'OR',
|
||||
) {}
|
||||
|
||||
where(column: string, operator: Operator, value?: SqlParam): this {
|
||||
const condition = this.query.buildCondition(column, operator, value);
|
||||
this.conditions.push({ condition, operator: 'AND' });
|
||||
return this;
|
||||
}
|
||||
|
||||
andWhere(column: string, operator: Operator, value?: SqlParam): this {
|
||||
const condition = this.query.buildCondition(column, operator, value);
|
||||
this.conditions.push({ condition, operator: 'AND' });
|
||||
return this;
|
||||
}
|
||||
|
||||
rawWhere(condition: string): this {
|
||||
this.conditions.push({ condition, operator: 'AND' });
|
||||
return this;
|
||||
}
|
||||
|
||||
orWhere(column: string, operator: Operator, value?: SqlParam): this {
|
||||
const condition = this.query.buildCondition(column, operator, value);
|
||||
this.conditions.push({ condition, operator: 'OR' });
|
||||
return this;
|
||||
}
|
||||
|
||||
end(): Query {
|
||||
const groupCondition = this.conditions
|
||||
.map((c, i) => (i === 0 ? c.condition : `${c.operator} ${c.condition}`))
|
||||
.join(' ');
|
||||
|
||||
this.query._addWhereCondition({
|
||||
condition: groupCondition,
|
||||
operator: this.groupOperator,
|
||||
isGroup: true,
|
||||
});
|
||||
|
||||
return this.query;
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to create a new query
|
||||
export function createQuery(client: ClickHouseClient): Query {
|
||||
return new Query(client);
|
||||
}
|
||||
|
||||
export function clix(client: ClickHouseClient): Query {
|
||||
return new Query(client);
|
||||
}
|
||||
|
||||
clix.exp = (expr: string | Query<any>) =>
|
||||
new Expression(expr instanceof Query ? expr.toSQL() : expr);
|
||||
clix.date = (date: string | Date, wrapper?: string) => {
|
||||
const dateStr = new Date(date).toISOString().slice(0, 10);
|
||||
return wrapper ? `${wrapper}(${dateStr})` : dateStr;
|
||||
};
|
||||
clix.datetime = (date: string | Date, wrapper?: string) => {
|
||||
const datetime = new Date(date).toISOString().slice(0, 19).replace('T', ' ');
|
||||
return wrapper ? `${wrapper}(${datetime})` : datetime;
|
||||
};
|
||||
clix.dynamicDatetime = (date: string | Date, interval: IInterval) => {
|
||||
if (interval === 'month' || interval === 'week') {
|
||||
return clix.date(date);
|
||||
}
|
||||
return clix.datetime(date);
|
||||
};
|
||||
|
||||
clix.toStartOf = (node: string, interval: IInterval) => {
|
||||
switch (interval) {
|
||||
case 'minute': {
|
||||
return `toStartOfMinute(${node})`;
|
||||
}
|
||||
case 'hour': {
|
||||
return `toStartOfHour(${node})`;
|
||||
}
|
||||
case 'day': {
|
||||
return `toStartOfDay(${node})`;
|
||||
}
|
||||
case 'week': {
|
||||
return `toStartOfWeek(${node})`;
|
||||
}
|
||||
case 'month': {
|
||||
return `toStartOfMonth(${node})`;
|
||||
}
|
||||
}
|
||||
};
|
||||
clix.toStartOfInterval = (
|
||||
node: string,
|
||||
interval: IInterval,
|
||||
origin: string | Date,
|
||||
) => {
|
||||
switch (interval) {
|
||||
case 'minute': {
|
||||
return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 minute, toDateTime(${clix.datetime(origin)}))`;
|
||||
}
|
||||
case 'hour': {
|
||||
return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 hour, toDateTime(${clix.datetime(origin)}))`;
|
||||
}
|
||||
case 'day': {
|
||||
return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 day, toDateTime(${clix.datetime(origin)}))`;
|
||||
}
|
||||
case 'week': {
|
||||
return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 week, toDateTime(${clix.datetime(origin)}))`;
|
||||
}
|
||||
case 'month': {
|
||||
return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 month, toDateTime(${clix.datetime(origin)}))`;
|
||||
}
|
||||
}
|
||||
};
|
||||
clix.toInterval = (node: string, interval: IInterval) => {
|
||||
switch (interval) {
|
||||
case 'minute': {
|
||||
return `toIntervalMinute(${node})`;
|
||||
}
|
||||
case 'hour': {
|
||||
return `toIntervalHour(${node})`;
|
||||
}
|
||||
case 'day': {
|
||||
return `toIntervalDay(${node})`;
|
||||
}
|
||||
case 'week': {
|
||||
return `toIntervalWeek(${node})`;
|
||||
}
|
||||
case 'month': {
|
||||
return `toIntervalMonth(${node})`;
|
||||
}
|
||||
}
|
||||
};
|
||||
clix.toDate = (node: string, interval: IInterval) => {
|
||||
switch (interval) {
|
||||
case 'week':
|
||||
case 'month': {
|
||||
return `toDate(${node})`;
|
||||
}
|
||||
default: {
|
||||
return `toDateTime(${node})`;
|
||||
}
|
||||
}
|
||||
};
|
||||
// Export types
|
||||
export type { SqlValue, SqlParam, Operator };
|
||||
@@ -17,31 +17,41 @@ import {
|
||||
import { createSqlBuilder } from '../sql-builder';
|
||||
|
||||
export function transformPropertyKey(property: string) {
|
||||
if (property.startsWith('properties.')) {
|
||||
if (property.includes('*')) {
|
||||
return property
|
||||
.replace(/^properties\./, '')
|
||||
.replace('.*.', '.%.')
|
||||
.replace(/\[\*\]$/, '.%')
|
||||
.replace(/\[\*\].?/, '.%.');
|
||||
}
|
||||
return `properties['${property.replace(/^properties\./, '')}']`;
|
||||
const propertyPatterns = ['properties', 'profile.properties'];
|
||||
const match = propertyPatterns.find((pattern) =>
|
||||
property.startsWith(`${pattern}.`),
|
||||
);
|
||||
|
||||
if (!match) {
|
||||
return property;
|
||||
}
|
||||
|
||||
return property;
|
||||
if (property.includes('*')) {
|
||||
return property
|
||||
.replace(/^properties\./, '')
|
||||
.replace('.*.', '.%.')
|
||||
.replace(/\[\*\]$/, '.%')
|
||||
.replace(/\[\*\].?/, '.%.');
|
||||
}
|
||||
|
||||
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
|
||||
}
|
||||
|
||||
export function getSelectPropertyKey(property: string) {
|
||||
if (property.startsWith('properties.')) {
|
||||
if (property.includes('*')) {
|
||||
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
|
||||
transformPropertyKey(property),
|
||||
)})))`;
|
||||
}
|
||||
return `properties['${property.replace(/^properties\./, '')}']`;
|
||||
const propertyPatterns = ['properties', 'profile.properties'];
|
||||
|
||||
const match = propertyPatterns.find((pattern) =>
|
||||
property.startsWith(`${pattern}.`),
|
||||
);
|
||||
if (!match) return property;
|
||||
|
||||
if (property.includes('*')) {
|
||||
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${escape(
|
||||
transformPropertyKey(property),
|
||||
)})))`;
|
||||
}
|
||||
|
||||
return property;
|
||||
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
|
||||
}
|
||||
|
||||
export function getChartSql({
|
||||
@@ -54,8 +64,16 @@ export function getChartSql({
|
||||
chartType,
|
||||
limit,
|
||||
}: IGetChartDataInput) {
|
||||
const { sb, join, getWhere, getFrom, getSelect, getOrderBy, getGroupBy } =
|
||||
createSqlBuilder();
|
||||
const {
|
||||
sb,
|
||||
join,
|
||||
getWhere,
|
||||
getFrom,
|
||||
getJoins,
|
||||
getSelect,
|
||||
getOrderBy,
|
||||
getGroupBy,
|
||||
} = createSqlBuilder();
|
||||
|
||||
sb.where = getEventFiltersWhereClause(event.filters);
|
||||
sb.where.projectId = `project_id = ${escape(projectId)}`;
|
||||
@@ -67,6 +85,14 @@ export function getChartSql({
|
||||
sb.select.label_0 = `'*' as label_0`;
|
||||
}
|
||||
|
||||
// const anyFilterOnProfile = event.filters.some((filter) =>
|
||||
// filter.name.startsWith('profile.properties.'),
|
||||
// );
|
||||
|
||||
// if (anyFilterOnProfile) {
|
||||
// sb.joins.profiles = 'JOIN profiles profile ON e.profile_id = profile.id';
|
||||
// }
|
||||
|
||||
sb.select.count = 'count(*) as count';
|
||||
switch (interval) {
|
||||
case 'minute': {
|
||||
@@ -149,10 +175,18 @@ export function getChartSql({
|
||||
ORDER BY profile_id, created_at DESC
|
||||
) as subQuery`;
|
||||
|
||||
return `${getSelect()} ${getFrom()} ${getGroupBy()} ${getOrderBy()}`;
|
||||
console.log(
|
||||
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
|
||||
);
|
||||
|
||||
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
|
||||
}
|
||||
|
||||
return `${getSelect()} ${getFrom()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
|
||||
console.log(
|
||||
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
|
||||
);
|
||||
|
||||
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
|
||||
}
|
||||
|
||||
export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
|
||||
@@ -161,7 +195,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
|
||||
const id = `f${index}`;
|
||||
const { name, value, operator } = filter;
|
||||
|
||||
if (value.length === 0) return;
|
||||
if (
|
||||
value.length === 0 &&
|
||||
operator !== 'isNull' &&
|
||||
operator !== 'isNotNull'
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (name === 'has_profile') {
|
||||
if (value.includes('true')) {
|
||||
@@ -172,7 +212,10 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (name.startsWith('properties.')) {
|
||||
if (
|
||||
name.startsWith('properties.') ||
|
||||
name.startsWith('profile.properties.')
|
||||
) {
|
||||
const propertyKey = getSelectPropertyKey(name);
|
||||
const isWildcard = propertyKey.includes('%');
|
||||
const whereFrom = getSelectPropertyKey(name);
|
||||
@@ -284,6 +327,23 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'isNull': {
|
||||
if (isWildcard) {
|
||||
where[id] = `arrayExists(x -> x = '' OR x IS NULL, ${whereFrom})`;
|
||||
} else {
|
||||
where[id] = `(${whereFrom} = '' OR ${whereFrom} IS NULL)`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'isNotNull': {
|
||||
if (isWildcard) {
|
||||
where[id] =
|
||||
`arrayExists(x -> x != '' AND x IS NOT NULL, ${whereFrom})`;
|
||||
} else {
|
||||
where[id] = `(${whereFrom} != '' AND ${whereFrom} IS NOT NULL)`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
switch (operator) {
|
||||
@@ -297,6 +357,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'isNull': {
|
||||
where[id] = `(${name} = '' OR ${name} IS NULL)`;
|
||||
break;
|
||||
}
|
||||
case 'isNotNull': {
|
||||
where[id] = `(${name} != '' AND ${name} IS NOT NULL)`;
|
||||
break;
|
||||
}
|
||||
case 'isNot': {
|
||||
if (value.length === 1) {
|
||||
where[id] = `${name} != ${escape(String(value[0]).trim())}`;
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
import { mergeDeepRight, uniq } from 'ramda';
|
||||
import { path, assocPath, last, mergeDeepRight, pick, uniq } from 'ramda';
|
||||
import { escape } from 'sqlstring';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
import { toDots } from '@openpanel/common';
|
||||
import { cacheable } from '@openpanel/redis';
|
||||
import { cacheable, getCache } from '@openpanel/redis';
|
||||
import type { IChartEventFilter } from '@openpanel/validation';
|
||||
|
||||
import { botBuffer, eventBuffer } from '../buffers';
|
||||
import { botBuffer, eventBuffer, sessionBuffer } from '../buffers';
|
||||
import {
|
||||
TABLE_NAMES,
|
||||
ch,
|
||||
chQuery,
|
||||
convertClickhouseDateToJs,
|
||||
formatClickhouseDate,
|
||||
} from '../clickhouse/client';
|
||||
import { type Query, clix } from '../clickhouse/query-builder';
|
||||
import type { EventMeta, Prisma } from '../prisma-client';
|
||||
import { db } from '../prisma-client';
|
||||
import { createSqlBuilder } from '../sql-builder';
|
||||
import { getEventFiltersWhereClause } from './chart.service';
|
||||
import type { IServiceProfile } from './profile.service';
|
||||
import { getProfiles, upsertProfile } from './profile.service';
|
||||
import type { IClickhouseProfile, IServiceProfile } from './profile.service';
|
||||
import {
|
||||
getProfiles,
|
||||
transformProfile,
|
||||
upsertProfile,
|
||||
} from './profile.service';
|
||||
|
||||
export type IImportedEvent = Omit<
|
||||
IClickhouseEvent,
|
||||
@@ -120,11 +126,11 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
|
||||
referrer: event.referrer,
|
||||
referrerName: event.referrer_name,
|
||||
referrerType: event.referrer_type,
|
||||
profile: event.profile,
|
||||
meta: event.meta,
|
||||
importedAt: event.imported_at ? new Date(event.imported_at) : undefined,
|
||||
sdkName: event.sdk_name,
|
||||
sdkVersion: event.sdk_version,
|
||||
profile: event.profile,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -246,24 +252,34 @@ export async function getEvents(
|
||||
const ids = events.map((e) => e.profile_id);
|
||||
const profiles = await getProfiles(ids, projectId);
|
||||
|
||||
const map = new Map<string, IServiceProfile>();
|
||||
for (const profile of profiles) {
|
||||
map.set(profile.id, profile);
|
||||
}
|
||||
|
||||
for (const event of events) {
|
||||
event.profile = profiles.find((p) => p.id === event.profile_id);
|
||||
event.profile = map.get(event.profile_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.meta && projectId) {
|
||||
const names = uniq(events.map((e) => e.name));
|
||||
const metas = await db.eventMeta.findMany({
|
||||
where: {
|
||||
name: {
|
||||
in: names,
|
||||
},
|
||||
projectId,
|
||||
const metas = await getCache(
|
||||
`event-metas-${projectId}`,
|
||||
60 * 5,
|
||||
async () => {
|
||||
return db.eventMeta.findMany({
|
||||
where: {
|
||||
projectId,
|
||||
},
|
||||
});
|
||||
},
|
||||
select: options.meta === true ? undefined : options.meta,
|
||||
});
|
||||
);
|
||||
const map = new Map<string, EventMeta>();
|
||||
for (const meta of metas) {
|
||||
map.set(meta.name, meta);
|
||||
}
|
||||
for (const event of events) {
|
||||
event.meta = metas.find((m) => m.name === event.name);
|
||||
event.meta = map.get(event.name);
|
||||
}
|
||||
}
|
||||
return events.map(transformEvent);
|
||||
@@ -339,7 +355,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
|
||||
sdk_version: payload.sdkVersion ?? '',
|
||||
};
|
||||
|
||||
await eventBuffer.add(event);
|
||||
await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);
|
||||
|
||||
return {
|
||||
document: event,
|
||||
@@ -350,7 +366,7 @@ export interface GetEventListOptions {
|
||||
projectId: string;
|
||||
profileId?: string;
|
||||
take: number;
|
||||
cursor?: number;
|
||||
cursor?: number | Date;
|
||||
events?: string[] | null;
|
||||
filters?: IChartEventFilter[];
|
||||
startDate?: Date;
|
||||
@@ -371,8 +387,13 @@ export async function getEventList({
|
||||
}: GetEventListOptions) {
|
||||
const { sb, getSql, join } = createSqlBuilder();
|
||||
|
||||
if (typeof cursor === 'number') {
|
||||
sb.offset = Math.max(0, (cursor ?? 0) * take);
|
||||
} else if (cursor instanceof Date) {
|
||||
sb.where.cursor = `created_at <= '${formatClickhouseDate(cursor)}'`;
|
||||
}
|
||||
|
||||
sb.limit = take;
|
||||
sb.offset = Math.max(0, (cursor ?? 0) * take);
|
||||
sb.where.projectId = `project_id = ${escape(projectId)}`;
|
||||
const select = mergeDeepRight(
|
||||
{
|
||||
@@ -380,6 +401,7 @@ export async function getEventList({
|
||||
name: true,
|
||||
deviceId: true,
|
||||
profileId: true,
|
||||
sessionId: true,
|
||||
projectId: true,
|
||||
createdAt: true,
|
||||
path: true,
|
||||
@@ -607,3 +629,320 @@ export async function getTopPages({
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
export interface IEventServiceGetList {
|
||||
projectId: string;
|
||||
profileId?: string;
|
||||
cursor?: Date;
|
||||
filters?: IChartEventFilter[];
|
||||
}
|
||||
|
||||
class EventService {
|
||||
constructor(private client: typeof ch) {}
|
||||
|
||||
query<T>({
|
||||
projectId,
|
||||
profileId,
|
||||
where,
|
||||
select,
|
||||
limit,
|
||||
orderBy,
|
||||
}: {
|
||||
projectId: string;
|
||||
profileId?: string;
|
||||
where?: {
|
||||
profile?: (query: Query<T>) => void;
|
||||
event?: (query: Query<T>) => void;
|
||||
session?: (query: Query<T>) => void;
|
||||
};
|
||||
select: {
|
||||
profile?: Partial<SelectHelper<IServiceProfile>>;
|
||||
event: Partial<SelectHelper<IServiceEvent>>;
|
||||
};
|
||||
limit?: number;
|
||||
orderBy?: keyof IClickhouseEvent;
|
||||
}) {
|
||||
const events = clix(this.client)
|
||||
.select<
|
||||
Partial<IClickhouseEvent> & {
|
||||
// profile
|
||||
profileId: string;
|
||||
profile_firstName: string;
|
||||
profile_lastName: string;
|
||||
profile_avatar: string;
|
||||
profile_isExternal: boolean;
|
||||
profile_createdAt: string;
|
||||
}
|
||||
>([
|
||||
select.event.id && 'e.id as id',
|
||||
select.event.deviceId && 'e.device_id as device_id',
|
||||
select.event.name && 'e.name as name',
|
||||
select.event.path && 'e.path as path',
|
||||
select.event.duration && 'e.duration as duration',
|
||||
select.event.country && 'e.country as country',
|
||||
select.event.city && 'e.city as city',
|
||||
select.event.os && 'e.os as os',
|
||||
select.event.browser && 'e.browser as browser',
|
||||
select.event.createdAt && 'e.created_at as created_at',
|
||||
select.event.projectId && 'e.project_id as project_id',
|
||||
'e.session_id as session_id',
|
||||
'e.profile_id as profile_id',
|
||||
])
|
||||
.from('events e')
|
||||
.where('project_id', '=', projectId)
|
||||
.when(!!where?.event, where?.event)
|
||||
// Do not limit if profileId, we will limit later since we need the "correct" profileId
|
||||
.when(!!limit && !profileId, (q) => q.limit(limit!))
|
||||
.orderBy('toDate(created_at)', 'DESC')
|
||||
.orderBy('created_at', 'DESC');
|
||||
|
||||
const sessions = clix(this.client)
|
||||
.select(['id as session_id', 'profile_id'])
|
||||
.from('sessions')
|
||||
.where('sign', '=', 1)
|
||||
.where('project_id', '=', projectId)
|
||||
.when(!!where?.session, where?.session)
|
||||
.when(!!profileId, (q) => q.where('profile_id', '=', profileId));
|
||||
|
||||
const profiles = clix(this.client)
|
||||
.select([
|
||||
'id',
|
||||
'any(created_at) as created_at',
|
||||
`any(nullIf(first_name, '')) as first_name`,
|
||||
`any(nullIf(last_name, '')) as last_name`,
|
||||
`any(nullIf(email, '')) as email`,
|
||||
`any(nullIf(avatar, '')) as avatar`,
|
||||
'last_value(is_external) as is_external',
|
||||
])
|
||||
.from('profiles')
|
||||
.where('project_id', '=', projectId)
|
||||
.where(
|
||||
'id',
|
||||
'IN',
|
||||
clix.exp(
|
||||
clix(this.client)
|
||||
.select(['profile_id'])
|
||||
.from(
|
||||
clix.exp(
|
||||
clix(this.client)
|
||||
.select(['profile_id'])
|
||||
.from('cte_sessions')
|
||||
.union(
|
||||
clix(this.client).select(['profile_id']).from('cte_events'),
|
||||
),
|
||||
),
|
||||
)
|
||||
.groupBy(['profile_id']),
|
||||
),
|
||||
)
|
||||
.groupBy(['id', 'project_id'])
|
||||
.when(!!where?.profile, where?.profile);
|
||||
|
||||
return clix(this.client)
|
||||
.with('cte_events', events)
|
||||
.with('cte_sessions', sessions)
|
||||
.with('cte_profiles', profiles)
|
||||
.select<
|
||||
Partial<IClickhouseEvent> & {
|
||||
// profile
|
||||
profileId: string;
|
||||
profile_firstName: string;
|
||||
profile_lastName: string;
|
||||
profile_avatar: string;
|
||||
profile_isExternal: boolean;
|
||||
profile_createdAt: string;
|
||||
}
|
||||
>([
|
||||
select.event.id && 'e.id as id',
|
||||
select.event.deviceId && 'e.device_id as device_id',
|
||||
select.event.name && 'e.name as name',
|
||||
select.event.path && 'e.path as path',
|
||||
select.event.duration && 'e.duration as duration',
|
||||
select.event.country && 'e.country as country',
|
||||
select.event.city && 'e.city as city',
|
||||
select.event.os && 'e.os as os',
|
||||
select.event.browser && 'e.browser as browser',
|
||||
select.event.createdAt && 'e.created_at as created_at',
|
||||
select.event.projectId && 'e.project_id as project_id',
|
||||
select.event.sessionId && 'e.session_id as session_id',
|
||||
select.event.profileId && 'e.profile_id as event_profile_id',
|
||||
// Profile
|
||||
select.profile?.id && 'p.id as profile_id',
|
||||
select.profile?.firstName && 'p.first_name as profile_first_name',
|
||||
select.profile?.lastName && 'p.last_name as profile_last_name',
|
||||
select.profile?.avatar && 'p.avatar as profile_avatar',
|
||||
select.profile?.isExternal && 'p.is_external as profile_is_external',
|
||||
select.profile?.createdAt && 'p.created_at as profile_created_at',
|
||||
select.profile?.email && 'p.email as profile_email',
|
||||
select.profile?.properties && 'p.properties as profile_properties',
|
||||
])
|
||||
.from('cte_events e')
|
||||
.leftJoin('cte_sessions s', 'e.session_id = s.session_id')
|
||||
.leftJoin(
|
||||
'cte_profiles p',
|
||||
's.profile_id = p.id AND p.is_external = true',
|
||||
)
|
||||
.when(!!profileId, (q) => {
|
||||
q.where('s.profile_id', '=', profileId);
|
||||
q.limit(limit!);
|
||||
});
|
||||
}
|
||||
|
||||
transformFromQuery(res: any[]) {
|
||||
return res
|
||||
.map((item) => {
|
||||
return Object.entries(item).reduce(
|
||||
(acc, [prop, val]) => {
|
||||
if (prop === 'event_profile_id' && val) {
|
||||
if (!item.profile_id) {
|
||||
return assocPath(['profile', 'id'], val, acc);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
prop.startsWith('profile_') &&
|
||||
!path(['profile', prop.replace('profile_', '')], acc)
|
||||
) {
|
||||
return assocPath(
|
||||
['profile', prop.replace('profile_', '')],
|
||||
val,
|
||||
acc,
|
||||
);
|
||||
}
|
||||
return assocPath([prop], val, acc);
|
||||
},
|
||||
{
|
||||
profile: {},
|
||||
} as IClickhouseEvent,
|
||||
);
|
||||
})
|
||||
.map(transformEvent);
|
||||
}
|
||||
|
||||
async getById({
|
||||
projectId,
|
||||
id,
|
||||
createdAt,
|
||||
}: {
|
||||
projectId: string;
|
||||
id: string;
|
||||
createdAt?: Date;
|
||||
}) {
|
||||
return clix(this.client)
|
||||
.select<IClickhouseEvent>(['*'])
|
||||
.from('events')
|
||||
.where('project_id', '=', projectId)
|
||||
.when(!!createdAt, (q) => {
|
||||
if (createdAt) {
|
||||
q.where('created_at', 'BETWEEN', [
|
||||
new Date(createdAt.getTime() - 1000),
|
||||
new Date(createdAt.getTime() + 1000),
|
||||
]);
|
||||
}
|
||||
})
|
||||
.where('id', '=', id)
|
||||
.limit(1)
|
||||
.execute()
|
||||
.then((res) => {
|
||||
if (!res[0]) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return transformEvent(res[0]);
|
||||
});
|
||||
}
|
||||
|
||||
async getList({
|
||||
projectId,
|
||||
profileId,
|
||||
cursor,
|
||||
filters,
|
||||
limit = 50,
|
||||
startDate,
|
||||
endDate,
|
||||
}: IEventServiceGetList & {
|
||||
limit?: number;
|
||||
startDate?: Date;
|
||||
endDate?: Date;
|
||||
}) {
|
||||
const date = cursor || new Date();
|
||||
const query = this.query({
|
||||
projectId,
|
||||
profileId,
|
||||
limit,
|
||||
orderBy: 'created_at',
|
||||
select: {
|
||||
event: {
|
||||
deviceId: true,
|
||||
profileId: true,
|
||||
id: true,
|
||||
name: true,
|
||||
createdAt: true,
|
||||
duration: true,
|
||||
country: true,
|
||||
city: true,
|
||||
os: true,
|
||||
browser: true,
|
||||
path: true,
|
||||
sessionId: true,
|
||||
},
|
||||
profile: {
|
||||
id: true,
|
||||
firstName: true,
|
||||
lastName: true,
|
||||
avatar: true,
|
||||
isExternal: true,
|
||||
},
|
||||
},
|
||||
where: {
|
||||
event: (q) => {
|
||||
if (startDate && endDate) {
|
||||
q.where('created_at', 'BETWEEN', [
|
||||
startDate ?? new Date(date.getTime() - 1000 * 60 * 60 * 24 * 3.5),
|
||||
cursor ?? endDate,
|
||||
]);
|
||||
} else {
|
||||
q.where('created_at', '<', date);
|
||||
}
|
||||
if (filters) {
|
||||
q.rawWhere(
|
||||
Object.values(getEventFiltersWhereClause(filters)).join(' AND '),
|
||||
);
|
||||
}
|
||||
},
|
||||
session: (q) => {
|
||||
if (startDate && endDate) {
|
||||
q.where('created_at', 'BETWEEN', [
|
||||
startDate ?? new Date(date.getTime() - 1000 * 60 * 60 * 24 * 3.5),
|
||||
endDate ?? date,
|
||||
]);
|
||||
} else {
|
||||
q.where('created_at', '<', date);
|
||||
}
|
||||
},
|
||||
},
|
||||
})
|
||||
.orderBy('toDate(created_at)', 'DESC')
|
||||
.orderBy('created_at', 'DESC');
|
||||
|
||||
const results = await query.execute();
|
||||
|
||||
// Current page items (middle chunk)
|
||||
const items = results.slice(0, limit);
|
||||
|
||||
// Check if there's a next page
|
||||
const hasNext = results.length >= limit;
|
||||
|
||||
return {
|
||||
items: this.transformFromQuery(items).map((item) => ({
|
||||
...item,
|
||||
projectId: projectId,
|
||||
})),
|
||||
meta: {
|
||||
next: hasNext ? last(items)?.created_at : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const eventService = new EventService(ch);
|
||||
|
||||
639
packages/db/src/services/overview.service.ts
Normal file
639
packages/db/src/services/overview.service.ts
Normal file
@@ -0,0 +1,639 @@
|
||||
import { average, sum } from '@openpanel/common';
|
||||
import { getCache } from '@openpanel/redis';
|
||||
import { type IChartEventFilter, zTimeInterval } from '@openpanel/validation';
|
||||
import { omit } from 'ramda';
|
||||
import { z } from 'zod';
|
||||
import { TABLE_NAMES, ch } from '../clickhouse/client';
|
||||
import { clix } from '../clickhouse/query-builder';
|
||||
import { getEventFiltersWhereClause } from './chart.service';
|
||||
|
||||
export const zGetMetricsInput = z.object({
|
||||
projectId: z.string(),
|
||||
filters: z.array(z.any()),
|
||||
startDate: z.string(),
|
||||
endDate: z.string(),
|
||||
interval: zTimeInterval,
|
||||
});
|
||||
|
||||
export type IGetMetricsInput = z.infer<typeof zGetMetricsInput>;
|
||||
|
||||
export const zGetTopPagesInput = z.object({
|
||||
projectId: z.string(),
|
||||
filters: z.array(z.any()),
|
||||
startDate: z.string(),
|
||||
endDate: z.string(),
|
||||
interval: zTimeInterval,
|
||||
cursor: z.number().optional(),
|
||||
limit: z.number().optional(),
|
||||
});
|
||||
|
||||
export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput>;
|
||||
|
||||
export const zGetTopEntryExitInput = z.object({
|
||||
projectId: z.string(),
|
||||
filters: z.array(z.any()),
|
||||
startDate: z.string(),
|
||||
endDate: z.string(),
|
||||
interval: zTimeInterval,
|
||||
mode: z.enum(['entry', 'exit']),
|
||||
cursor: z.number().optional(),
|
||||
limit: z.number().optional(),
|
||||
});
|
||||
|
||||
export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput>;
|
||||
|
||||
export const zGetTopGenericInput = z.object({
|
||||
projectId: z.string(),
|
||||
filters: z.array(z.any()),
|
||||
startDate: z.string(),
|
||||
endDate: z.string(),
|
||||
interval: zTimeInterval,
|
||||
column: z.enum([
|
||||
// Referrers
|
||||
'referrer',
|
||||
'referrer_name',
|
||||
'referrer_type',
|
||||
'utm_source',
|
||||
'utm_medium',
|
||||
'utm_campaign',
|
||||
'utm_term',
|
||||
'utm_content',
|
||||
// Geo
|
||||
'region',
|
||||
'country',
|
||||
'city',
|
||||
// Device
|
||||
'device',
|
||||
'brand',
|
||||
'model',
|
||||
'browser',
|
||||
'browser_version',
|
||||
'os',
|
||||
'os_version',
|
||||
]),
|
||||
cursor: z.number().optional(),
|
||||
limit: z.number().optional(),
|
||||
});
|
||||
|
||||
export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput>;
|
||||
|
||||
export class OverviewService {
|
||||
private pendingQueries: Map<string, Promise<number | null>> = new Map();
|
||||
|
||||
constructor(private client: typeof ch) {}
|
||||
|
||||
isPageFilter(filters: IChartEventFilter[]) {
|
||||
return filters.some((filter) => filter.name === 'path' && filter.value);
|
||||
}
|
||||
|
||||
getTotalSessions({
|
||||
projectId,
|
||||
startDate,
|
||||
endDate,
|
||||
filters,
|
||||
}: {
|
||||
projectId: string;
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
filters: IChartEventFilter[];
|
||||
}) {
|
||||
const where = this.getRawWhereClause('sessions', filters);
|
||||
const key = `total_sessions_${projectId}_${startDate}_${endDate}_${JSON.stringify(filters)}`;
|
||||
|
||||
// Check if there's already a pending query for this key
|
||||
const pendingQuery = this.pendingQueries.get(key);
|
||||
if (pendingQuery) {
|
||||
return pendingQuery.then((res) => res ?? 0);
|
||||
}
|
||||
|
||||
// Create new query promise and store it
|
||||
const queryPromise = getCache(key, 15, async () => {
|
||||
try {
|
||||
const result = await clix(this.client)
|
||||
.select<{
|
||||
total_sessions: number;
|
||||
}>(['sum(sign) as total_sessions'])
|
||||
.from(TABLE_NAMES.sessions, true)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.rawWhere(where)
|
||||
.having('sum(sign)', '>', 0)
|
||||
.execute();
|
||||
return result?.[0]?.total_sessions ?? 0;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
this.pendingQueries.set(key, queryPromise);
|
||||
return queryPromise;
|
||||
}
|
||||
|
||||
getMetrics({
|
||||
projectId,
|
||||
filters,
|
||||
startDate,
|
||||
endDate,
|
||||
interval,
|
||||
}: IGetMetricsInput): Promise<{
|
||||
metrics: {
|
||||
bounce_rate: number;
|
||||
unique_visitors: number;
|
||||
total_sessions: number;
|
||||
avg_session_duration: number;
|
||||
total_screen_views: number;
|
||||
views_per_session: number;
|
||||
};
|
||||
series: {
|
||||
date: string;
|
||||
bounce_rate: number;
|
||||
unique_visitors: number;
|
||||
total_sessions: number;
|
||||
avg_session_duration: number;
|
||||
total_screen_views: number;
|
||||
views_per_session: number;
|
||||
}[];
|
||||
}> {
|
||||
const where = this.getRawWhereClause('sessions', filters);
|
||||
if (this.isPageFilter(filters)) {
|
||||
// Session aggregation with bounce rates
|
||||
const sessionAggQuery = clix(this.client)
|
||||
.select([
|
||||
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
|
||||
'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions, true)
|
||||
.where('sign', '=', 1)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.rawWhere(where)
|
||||
.groupBy(['date'])
|
||||
.rollup()
|
||||
.orderBy('date', 'ASC');
|
||||
|
||||
// Overall unique visitors
|
||||
const overallUniqueVisitorsQuery = clix(this.client)
|
||||
.select([
|
||||
'uniq(profile_id) AS unique_visitors',
|
||||
'uniq(session_id) AS total_sessions',
|
||||
])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.rawWhere(this.getRawWhereClause('events', filters));
|
||||
|
||||
return clix(this.client)
|
||||
.with('session_agg', sessionAggQuery)
|
||||
.with(
|
||||
'overall_bounce_rate',
|
||||
clix(this.client)
|
||||
.select(['bounce_rate'])
|
||||
.from('session_agg')
|
||||
.where('date', '=', clix.exp("'1970-01-01 00:00:00'")),
|
||||
)
|
||||
.with(
|
||||
'daily_stats',
|
||||
clix(this.client)
|
||||
.select(['date', 'bounce_rate'])
|
||||
.from('session_agg')
|
||||
.where('date', '!=', clix.exp("'1970-01-01 00:00:00'")),
|
||||
)
|
||||
.with('overall_unique_visitors', overallUniqueVisitorsQuery)
|
||||
.select<{
|
||||
date: string;
|
||||
bounce_rate: number;
|
||||
unique_visitors: number;
|
||||
total_sessions: number;
|
||||
avg_session_duration: number;
|
||||
total_screen_views: number;
|
||||
views_per_session: number;
|
||||
overall_unique_visitors: number;
|
||||
overall_total_sessions: number;
|
||||
overall_bounce_rate: number;
|
||||
}>([
|
||||
`${clix.toStartOfInterval('e.created_at', interval, startDate)} AS date`,
|
||||
'ds.bounce_rate as bounce_rate',
|
||||
'uniq(e.profile_id) AS unique_visitors',
|
||||
'uniq(e.session_id) AS total_sessions',
|
||||
'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
|
||||
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
|
||||
'count(*) AS total_screen_views',
|
||||
'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
|
||||
'(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
|
||||
'(SELECT total_sessions FROM overall_unique_visitors) AS overall_total_sessions',
|
||||
'(SELECT bounce_rate FROM overall_bounce_rate) AS overall_bounce_rate',
|
||||
])
|
||||
.from(`${TABLE_NAMES.events} AS e`)
|
||||
.leftJoin(
|
||||
'daily_stats AS ds',
|
||||
`${clix.toStartOfInterval('e.created_at', interval, startDate)} = ds.date`,
|
||||
)
|
||||
.where('e.project_id', '=', projectId)
|
||||
.where('e.name', '=', 'screen_view')
|
||||
.where('e.created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.rawWhere(this.getRawWhereClause('events', filters))
|
||||
.groupBy(['date', 'ds.bounce_rate'])
|
||||
.orderBy('date', 'ASC')
|
||||
.fill(
|
||||
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
|
||||
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
|
||||
clix.toInterval('1', interval),
|
||||
)
|
||||
.transform({
|
||||
date: (item) => new Date(item.date).toISOString(),
|
||||
})
|
||||
.execute()
|
||||
.then((res) => {
|
||||
const anyRowWithData = res.find(
|
||||
(item) =>
|
||||
item.overall_bounce_rate !== null ||
|
||||
item.overall_total_sessions !== null ||
|
||||
item.overall_unique_visitors !== null,
|
||||
);
|
||||
return {
|
||||
metrics: {
|
||||
bounce_rate: anyRowWithData?.overall_bounce_rate ?? 0,
|
||||
unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0,
|
||||
total_sessions: anyRowWithData?.overall_total_sessions ?? 0,
|
||||
avg_session_duration: average(
|
||||
res.map((item) => item.avg_session_duration),
|
||||
),
|
||||
total_screen_views: sum(
|
||||
res.map((item) => item.total_screen_views),
|
||||
),
|
||||
views_per_session: average(
|
||||
res.map((item) => item.views_per_session),
|
||||
),
|
||||
},
|
||||
series: res.map(
|
||||
omit([
|
||||
'overall_bounce_rate',
|
||||
'overall_unique_visitors',
|
||||
'overall_total_sessions',
|
||||
]),
|
||||
),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
const query = clix(this.client)
|
||||
.select<{
|
||||
date: string;
|
||||
bounce_rate: number;
|
||||
unique_visitors: number;
|
||||
total_sessions: number;
|
||||
avg_session_duration: number;
|
||||
total_screen_views: number;
|
||||
views_per_session: number;
|
||||
}>([
|
||||
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
|
||||
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
|
||||
'uniqIf(profile_id, sign > 0) AS unique_visitors',
|
||||
'sum(sign) AS total_sessions',
|
||||
'round(avgIf(duration, duration > 0 AND sign > 0), 2) / 1000 AS _avg_session_duration',
|
||||
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
|
||||
'sum(sign * screen_view_count) AS total_screen_views',
|
||||
'round(sum(sign * screen_view_count) * 1.0 / sum(sign), 2) AS views_per_session',
|
||||
])
|
||||
.from('sessions')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.where('project_id', '=', projectId)
|
||||
.rawWhere(where)
|
||||
.groupBy(['date'])
|
||||
.having('sum(sign)', '>', 0)
|
||||
.rollup()
|
||||
.orderBy('date', 'ASC')
|
||||
.fill(
|
||||
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
|
||||
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
|
||||
clix.toInterval('1', interval),
|
||||
)
|
||||
.transform({
|
||||
date: (item) => new Date(item.date).toISOString(),
|
||||
});
|
||||
|
||||
return query.execute().then((res) => {
|
||||
// First row is the rollup row containing the total values
|
||||
return {
|
||||
metrics: {
|
||||
bounce_rate: res[0]?.bounce_rate ?? 0,
|
||||
unique_visitors: res[0]?.unique_visitors ?? 0,
|
||||
total_sessions: res[0]?.total_sessions ?? 0,
|
||||
avg_session_duration: res[0]?.avg_session_duration ?? 0,
|
||||
total_screen_views: res[0]?.total_screen_views ?? 0,
|
||||
views_per_session: res[0]?.views_per_session ?? 0,
|
||||
},
|
||||
series: res
|
||||
.slice(1)
|
||||
.map(omit(['overall_bounce_rate', 'overall_unique_visitors'])),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
getRawWhereClause(type: 'events' | 'sessions', filters: IChartEventFilter[]) {
|
||||
const where = getEventFiltersWhereClause(
|
||||
filters.map((item) => {
|
||||
if (type === 'sessions') {
|
||||
if (item.name === 'path') {
|
||||
return { ...item, name: 'entry_path' };
|
||||
}
|
||||
if (item.name === 'origin') {
|
||||
return { ...item, name: 'entry_origin' };
|
||||
}
|
||||
if (item.name.startsWith('properties.__query.utm_')) {
|
||||
return {
|
||||
...item,
|
||||
name: item.name.replace('properties.__query.utm_', 'utm_'),
|
||||
};
|
||||
}
|
||||
return item;
|
||||
}
|
||||
return item;
|
||||
}),
|
||||
// .filter((item) => {
|
||||
// if (this.isPageFilter(filters) && type === 'sessions') {
|
||||
// return item.name !== 'entry_path' && item.name !== 'entry_origin';
|
||||
// }
|
||||
// return true;
|
||||
// }),
|
||||
);
|
||||
|
||||
return Object.values(where).join(' AND ');
|
||||
}
|
||||
|
||||
async getTopPages({
|
||||
projectId,
|
||||
filters,
|
||||
startDate,
|
||||
endDate,
|
||||
cursor = 1,
|
||||
limit = 10,
|
||||
}: IGetTopPagesInput) {
|
||||
const pageStatsQuery = clix(this.client)
|
||||
.select([
|
||||
'origin',
|
||||
'path',
|
||||
'uniq(session_id) as count',
|
||||
'round(avg(duration)/1000, 2) as avg_duration',
|
||||
])
|
||||
.from(TABLE_NAMES.events, false)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('name', '=', 'screen_view')
|
||||
.where('path', '!=', '')
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.groupBy(['origin', 'path'])
|
||||
.orderBy('count', 'DESC')
|
||||
.limit(limit)
|
||||
.offset((cursor - 1) * limit);
|
||||
|
||||
const bounceStatsQuery = clix(this.client)
|
||||
.select([
|
||||
'entry_path',
|
||||
'entry_origin',
|
||||
'coalesce(round(countIf(is_bounce = 1 AND sign = 1) * 100.0 / countIf(sign = 1), 2), 0) as bounce_rate',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions, true)
|
||||
.where('sign', '=', 1)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.groupBy(['entry_path', 'entry_origin']);
|
||||
|
||||
pageStatsQuery.rawWhere(this.getRawWhereClause('events', filters));
|
||||
bounceStatsQuery.rawWhere(this.getRawWhereClause('sessions', filters));
|
||||
|
||||
const mainQuery = clix(this.client)
|
||||
.with('page_stats', pageStatsQuery)
|
||||
.with('bounce_stats', bounceStatsQuery)
|
||||
.select<{
|
||||
origin: string;
|
||||
path: string;
|
||||
avg_duration: number;
|
||||
bounce_rate: number;
|
||||
sessions: number;
|
||||
}>([
|
||||
'p.origin',
|
||||
'p.path',
|
||||
'p.avg_duration',
|
||||
'p.count as sessions',
|
||||
'b.bounce_rate',
|
||||
])
|
||||
.from('page_stats p', false)
|
||||
.leftJoin(
|
||||
'bounce_stats b',
|
||||
'p.path = b.entry_path AND p.origin = b.entry_origin',
|
||||
)
|
||||
.orderBy('sessions', 'DESC')
|
||||
.limit(limit);
|
||||
|
||||
const totalSessions = await this.getTotalSessions({
|
||||
projectId,
|
||||
startDate,
|
||||
endDate,
|
||||
filters,
|
||||
});
|
||||
|
||||
return mainQuery.execute();
|
||||
}
|
||||
|
||||
async getTopEntryExit({
|
||||
projectId,
|
||||
filters,
|
||||
startDate,
|
||||
endDate,
|
||||
mode,
|
||||
cursor = 1,
|
||||
limit = 10,
|
||||
}: IGetTopEntryExitInput) {
|
||||
const where = this.getRawWhereClause('sessions', filters);
|
||||
|
||||
const distinctSessionQuery = this.getDistinctSessions({
|
||||
projectId,
|
||||
filters,
|
||||
startDate,
|
||||
endDate,
|
||||
});
|
||||
|
||||
const offset = (cursor - 1) * limit;
|
||||
|
||||
const query = clix(this.client)
|
||||
.select<{
|
||||
origin: string;
|
||||
path: string;
|
||||
avg_duration: number;
|
||||
bounce_rate: number;
|
||||
sessions: number;
|
||||
}>([
|
||||
`${mode}_origin AS origin`,
|
||||
`${mode}_path AS path`,
|
||||
'round(avg(duration * sign)/1000, 2) as avg_duration',
|
||||
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
|
||||
'sum(sign) as sessions',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions, true)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.rawWhere(where)
|
||||
.groupBy([`${mode}_origin`, `${mode}_path`])
|
||||
.having('sum(sign)', '>', 0)
|
||||
.orderBy('sessions', 'DESC')
|
||||
.limit(limit)
|
||||
.offset(offset);
|
||||
|
||||
let mainQuery = query;
|
||||
|
||||
if (this.isPageFilter(filters)) {
|
||||
mainQuery = clix(this.client)
|
||||
.with('distinct_sessions', distinctSessionQuery)
|
||||
.merge(query)
|
||||
.where(
|
||||
'id',
|
||||
'IN',
|
||||
clix.exp('(SELECT session_id FROM distinct_sessions)'),
|
||||
);
|
||||
}
|
||||
|
||||
const totalSessions = await this.getTotalSessions({
|
||||
projectId,
|
||||
startDate,
|
||||
endDate,
|
||||
filters,
|
||||
});
|
||||
|
||||
return mainQuery.execute();
|
||||
}
|
||||
|
||||
private getDistinctSessions({
|
||||
projectId,
|
||||
filters,
|
||||
startDate,
|
||||
endDate,
|
||||
}: {
|
||||
projectId: string;
|
||||
filters: IChartEventFilter[];
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
}) {
|
||||
return clix(this.client)
|
||||
.select(['DISTINCT session_id'])
|
||||
.from(TABLE_NAMES.events)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.rawWhere(this.getRawWhereClause('events', filters));
|
||||
}
|
||||
|
||||
async getTopGeneric({
|
||||
projectId,
|
||||
filters,
|
||||
startDate,
|
||||
endDate,
|
||||
column,
|
||||
cursor = 1,
|
||||
limit = 10,
|
||||
}: IGetTopGenericInput) {
|
||||
const distinctSessionQuery = this.getDistinctSessions({
|
||||
projectId,
|
||||
filters,
|
||||
startDate,
|
||||
endDate,
|
||||
});
|
||||
|
||||
const prefixColumn = (() => {
|
||||
switch (column) {
|
||||
case 'region':
|
||||
return 'country';
|
||||
case 'city':
|
||||
return 'country';
|
||||
case 'browser_version':
|
||||
return 'browser';
|
||||
case 'os_version':
|
||||
return 'os';
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
const offset = (cursor - 1) * limit;
|
||||
|
||||
const query = clix(this.client)
|
||||
.select<{
|
||||
prefix?: string;
|
||||
name: string;
|
||||
sessions: number;
|
||||
bounce_rate: number;
|
||||
avg_session_duration: number;
|
||||
}>([
|
||||
prefixColumn && `${prefixColumn} as prefix`,
|
||||
`nullIf(${column}, '') as name`,
|
||||
'sum(sign) as sessions',
|
||||
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) AS bounce_rate',
|
||||
'round(avgIf(duration, duration > 0 AND sign > 0), 2)/1000 AS avg_session_duration',
|
||||
])
|
||||
.from(TABLE_NAMES.sessions, true)
|
||||
.where('project_id', '=', projectId)
|
||||
.where('created_at', 'BETWEEN', [
|
||||
clix.datetime(startDate),
|
||||
clix.datetime(endDate),
|
||||
])
|
||||
.groupBy([prefixColumn, column].filter(Boolean))
|
||||
.having('sum(sign)', '>', 0)
|
||||
.orderBy('sessions', 'DESC')
|
||||
.limit(limit)
|
||||
.offset(offset);
|
||||
|
||||
let mainQuery = query;
|
||||
|
||||
if (this.isPageFilter(filters)) {
|
||||
mainQuery = clix(this.client)
|
||||
.with('distinct_sessions', distinctSessionQuery)
|
||||
.merge(query)
|
||||
.where(
|
||||
'id',
|
||||
'IN',
|
||||
clix.exp('(SELECT session_id FROM distinct_sessions)'),
|
||||
);
|
||||
} else {
|
||||
mainQuery.rawWhere(this.getRawWhereClause('sessions', filters));
|
||||
}
|
||||
|
||||
const [res, totalSessions] = await Promise.all([
|
||||
mainQuery.execute(),
|
||||
this.getTotalSessions({
|
||||
projectId,
|
||||
startDate,
|
||||
endDate,
|
||||
filters,
|
||||
}),
|
||||
]);
|
||||
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
export const overviewService = new OverviewService(ch);
|
||||
@@ -49,7 +49,17 @@ export async function getProfileById(id: string, projectId: string) {
|
||||
}
|
||||
|
||||
const [profile] = await chQuery<IClickhouseProfile>(
|
||||
`SELECT * FROM ${TABLE_NAMES.profiles} WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`,
|
||||
`SELECT
|
||||
id,
|
||||
project_id,
|
||||
last_value(nullIf(first_name, '')) as first_name,
|
||||
last_value(nullIf(last_name, '')) as last_name,
|
||||
last_value(nullIf(email, '')) as email,
|
||||
last_value(nullIf(avatar, '')) as avatar,
|
||||
last_value(is_external) as is_external,
|
||||
last_value(properties) as properties,
|
||||
last_value(created_at) as created_at
|
||||
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
|
||||
);
|
||||
|
||||
if (!profile) {
|
||||
@@ -59,7 +69,7 @@ export async function getProfileById(id: string, projectId: string) {
|
||||
return transformProfile(profile);
|
||||
}
|
||||
|
||||
export const getProfileByIdCached = cacheable(getProfileById, 60 * 30);
|
||||
export const getProfileByIdCached = getProfileById; //cacheable(getProfileById, 60 * 30);
|
||||
|
||||
interface GetProfileListOptions {
|
||||
projectId: string;
|
||||
@@ -77,11 +87,21 @@ export async function getProfiles(ids: string[], projectId: string) {
|
||||
}
|
||||
|
||||
const data = await chQuery<IClickhouseProfile>(
|
||||
`SELECT id, first_name, last_name, email, avatar, is_external, properties, created_at
|
||||
FROM ${TABLE_NAMES.profiles} FINAL
|
||||
`SELECT
|
||||
id,
|
||||
project_id,
|
||||
any(nullIf(first_name, '')) as first_name,
|
||||
any(nullIf(last_name, '')) as last_name,
|
||||
any(nullIf(email, '')) as email,
|
||||
any(nullIf(avatar, '')) as avatar,
|
||||
last_value(is_external) as is_external,
|
||||
any(properties) as properties,
|
||||
any(created_at) as created_at
|
||||
FROM ${TABLE_NAMES.profiles}
|
||||
WHERE
|
||||
project_id = ${escape(projectId)} AND
|
||||
id IN (${filteredIds.map((id) => escape(id)).join(',')})
|
||||
GROUP BY id, project_id
|
||||
`,
|
||||
);
|
||||
|
||||
|
||||
41
packages/db/src/services/session.service.ts
Normal file
41
packages/db/src/services/session.service.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
export type IClickhouseSession = {
|
||||
id: string;
|
||||
profile_id: string;
|
||||
event_count: number;
|
||||
screen_view_count: number;
|
||||
screen_views: string[];
|
||||
entry_path: string;
|
||||
entry_origin: string;
|
||||
exit_path: string;
|
||||
exit_origin: string;
|
||||
created_at: string;
|
||||
ended_at: string;
|
||||
referrer: string;
|
||||
referrer_name: string;
|
||||
referrer_type: string;
|
||||
os: string;
|
||||
os_version: string;
|
||||
browser: string;
|
||||
browser_version: string;
|
||||
device: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
country: string;
|
||||
region: string;
|
||||
city: string;
|
||||
longitude: number | null;
|
||||
latitude: number | null;
|
||||
is_bounce: boolean;
|
||||
project_id: string;
|
||||
device_id: string;
|
||||
duration: number;
|
||||
utm_medium: string;
|
||||
utm_source: string;
|
||||
utm_campaign: string;
|
||||
utm_content: string;
|
||||
utm_term: string;
|
||||
revenue: number;
|
||||
sign: 1 | 0;
|
||||
version: number;
|
||||
properties: Record<string, string>;
|
||||
};
|
||||
@@ -7,6 +7,7 @@ export interface SqlBuilderObject {
|
||||
groupBy: Record<string, string>;
|
||||
orderBy: Record<string, string>;
|
||||
from: string;
|
||||
joins: Record<string, string>;
|
||||
limit: number | undefined;
|
||||
offset: number | undefined;
|
||||
}
|
||||
@@ -17,11 +18,12 @@ export function createSqlBuilder() {
|
||||
|
||||
const sb: SqlBuilderObject = {
|
||||
where: {},
|
||||
from: TABLE_NAMES.events,
|
||||
from: `${TABLE_NAMES.events} e`,
|
||||
select: {},
|
||||
groupBy: {},
|
||||
orderBy: {},
|
||||
having: {},
|
||||
joins: {},
|
||||
limit: undefined,
|
||||
offset: undefined,
|
||||
};
|
||||
@@ -39,6 +41,8 @@ export function createSqlBuilder() {
|
||||
Object.keys(sb.orderBy).length ? `ORDER BY ${join(sb.orderBy, ', ')}` : '';
|
||||
const getLimit = () => (sb.limit ? `LIMIT ${sb.limit}` : '');
|
||||
const getOffset = () => (sb.offset ? `OFFSET ${sb.offset}` : '');
|
||||
const getJoins = () =>
|
||||
Object.keys(sb.joins).length ? join(sb.joins, ' ') : '';
|
||||
|
||||
return {
|
||||
sb,
|
||||
@@ -49,10 +53,12 @@ export function createSqlBuilder() {
|
||||
getGroupBy,
|
||||
getOrderBy,
|
||||
getHaving,
|
||||
getJoins,
|
||||
getSql: () => {
|
||||
const sql = [
|
||||
getSelect(),
|
||||
getFrom(),
|
||||
getJoins(),
|
||||
getWhere(),
|
||||
getGroupBy(),
|
||||
getHaving(),
|
||||
|
||||
Reference in New Issue
Block a user