zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feature(dashboard): refactor overview

Browse Source 
fix(lint)
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-03-20 09:28:54 +01:00
committed by Carl-Gerhard Lindesvärd
parent b035c0d586
commit a1eb4a296f
83 changed files with 59167 additions and 32403 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
packages/db/src/services/chart.service.ts
View File

@@ -17,31 +17,41 @@ import {
import { createSqlBuilder } from '../sql-builder';
export function transformPropertyKey(property: string) {
if (property.startsWith('properties.')) {
if (property.includes('*')) {
return property
.replace(/^properties\./, '')
.replace('.*.', '.%.')
.replace(/\[\*\]$/, '.%')
.replace(/\[\*\].?/, '.%.');
}
return `properties['${property.replace(/^properties\./, '')}']`;
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
);
if (!match) {
return property;
}
return property;
if (property.includes('*')) {
return property
.replace(/^properties\./, '')
.replace('.*.', '.%.')
.replace(/\[\*\]$/, '.%')
.replace(/\[\*\].?/, '.%.');
}
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
export function getSelectPropertyKey(property: string) {
if (property.startsWith('properties.')) {
if (property.includes('*')) {
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
transformPropertyKey(property),
)})))`;
}
return `properties['${property.replace(/^properties\./, '')}']`;
const propertyPatterns = ['properties', 'profile.properties'];
const match = propertyPatterns.find((pattern) =>
property.startsWith(`${pattern}.`),
);
if (!match) return property;
if (property.includes('*')) {
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${escape(
transformPropertyKey(property),
)})))`;
}
return property;
return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
}
export function getChartSql({
@@ -54,8 +64,16 @@ export function getChartSql({
chartType,
limit,
}: IGetChartDataInput) {
const { sb, join, getWhere, getFrom, getSelect, getOrderBy, getGroupBy } =
createSqlBuilder();
const {
sb,
join,
getWhere,
getFrom,
getJoins,
getSelect,
getOrderBy,
getGroupBy,
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.projectId = `project_id = ${escape(projectId)}`;
@@ -67,6 +85,14 @@ export function getChartSql({
sb.select.label_0 = `'*' as label_0`;
}
// const anyFilterOnProfile = event.filters.some((filter) =>
// filter.name.startsWith('profile.properties.'),
// );
// if (anyFilterOnProfile) {
// sb.joins.profiles = 'JOIN profiles profile ON e.profile_id = profile.id';
// }
sb.select.count = 'count(*) as count';
switch (interval) {
case 'minute': {
@@ -149,10 +175,18 @@ export function getChartSql({
ORDER BY profile_id, created_at DESC
) as subQuery`;
return `${getSelect()} ${getFrom()} ${getGroupBy()} ${getOrderBy()}`;
console.log(
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
);
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
}
return `${getSelect()} ${getFrom()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
console.log(
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
);
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
}
export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
@@ -161,7 +195,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
const id = `f${index}`;
const { name, value, operator } = filter;
if (value.length === 0) return;
if (
value.length === 0 &&
operator !== 'isNull' &&
operator !== 'isNotNull'
) {
return;
}
if (name === 'has_profile') {
if (value.includes('true')) {
@@ -172,7 +212,10 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
return;
}
if (name.startsWith('properties.')) {
if (
name.startsWith('properties.') ||
name.startsWith('profile.properties.')
) {
const propertyKey = getSelectPropertyKey(name);
const isWildcard = propertyKey.includes('%');
const whereFrom = getSelectPropertyKey(name);
@@ -284,6 +327,23 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
}
break;
}
case 'isNull': {
if (isWildcard) {
where[id] = `arrayExists(x -> x = '' OR x IS NULL, ${whereFrom})`;
} else {
where[id] = `(${whereFrom} = '' OR ${whereFrom} IS NULL)`;
}
break;
}
case 'isNotNull': {
if (isWildcard) {
where[id] =
`arrayExists(x -> x != '' AND x IS NOT NULL, ${whereFrom})`;
} else {
where[id] = `(${whereFrom} != '' AND ${whereFrom} IS NOT NULL)`;
}
break;
}
}
} else {
switch (operator) {
@@ -297,6 +357,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
}
break;
}
case 'isNull': {
where[id] = `(${name} = '' OR ${name} IS NULL)`;
break;
}
case 'isNotNull': {
where[id] = `(${name} != '' AND ${name} IS NOT NULL)`;
break;
}
case 'isNot': {
if (value.length === 1) {
where[id] = `${name} != ${escape(String(value[0]).trim())}`;

379
packages/db/src/services/event.service.ts
View File

@@ -1,24 +1,30 @@
import { mergeDeepRight, uniq } from 'ramda';
import { path, assocPath, last, mergeDeepRight, pick, uniq } from 'ramda';
import { escape } from 'sqlstring';
import { v4 as uuid } from 'uuid';
import { toDots } from '@openpanel/common';
import { cacheable } from '@openpanel/redis';
import { cacheable, getCache } from '@openpanel/redis';
import type { IChartEventFilter } from '@openpanel/validation';
import { botBuffer, eventBuffer } from '../buffers';
import { botBuffer, eventBuffer, sessionBuffer } from '../buffers';
import {
TABLE_NAMES,
ch,
chQuery,
convertClickhouseDateToJs,
formatClickhouseDate,
} from '../clickhouse/client';
import { type Query, clix } from '../clickhouse/query-builder';
import type { EventMeta, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
import { createSqlBuilder } from '../sql-builder';
import { getEventFiltersWhereClause } from './chart.service';
import type { IServiceProfile } from './profile.service';
import { getProfiles, upsertProfile } from './profile.service';
import type { IClickhouseProfile, IServiceProfile } from './profile.service';
import {
getProfiles,
transformProfile,
upsertProfile,
} from './profile.service';
export type IImportedEvent = Omit<
IClickhouseEvent,
@@ -120,11 +126,11 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
referrer: event.referrer,
referrerName: event.referrer_name,
referrerType: event.referrer_type,
profile: event.profile,
meta: event.meta,
importedAt: event.imported_at ? new Date(event.imported_at) : undefined,
sdkName: event.sdk_name,
sdkVersion: event.sdk_version,
profile: event.profile,
};
}
@@ -246,24 +252,34 @@ export async function getEvents(
const ids = events.map((e) => e.profile_id);
const profiles = await getProfiles(ids, projectId);
const map = new Map<string, IServiceProfile>();
for (const profile of profiles) {
map.set(profile.id, profile);
}
for (const event of events) {
event.profile = profiles.find((p) => p.id === event.profile_id);
event.profile = map.get(event.profile_id);
}
}
if (options.meta && projectId) {
const names = uniq(events.map((e) => e.name));
const metas = await db.eventMeta.findMany({
where: {
name: {
in: names,
},
projectId,
const metas = await getCache(
`event-metas-${projectId}`,
60 * 5,
async () => {
return db.eventMeta.findMany({
where: {
projectId,
},
});
},
select: options.meta === true ? undefined : options.meta,
});
);
const map = new Map<string, EventMeta>();
for (const meta of metas) {
map.set(meta.name, meta);
}
for (const event of events) {
event.meta = metas.find((m) => m.name === event.name);
event.meta = map.get(event.name);
}
}
return events.map(transformEvent);
@@ -339,7 +355,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
sdk_version: payload.sdkVersion ?? '',
};
await eventBuffer.add(event);
await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);
return {
document: event,
@@ -350,7 +366,7 @@ export interface GetEventListOptions {
projectId: string;
profileId?: string;
take: number;
cursor?: number;
cursor?: number | Date;
events?: string[] | null;
filters?: IChartEventFilter[];
startDate?: Date;
@@ -371,8 +387,13 @@ export async function getEventList({
}: GetEventListOptions) {
const { sb, getSql, join } = createSqlBuilder();
if (typeof cursor === 'number') {
sb.offset = Math.max(0, (cursor ?? 0) * take);
} else if (cursor instanceof Date) {
sb.where.cursor = `created_at <= '${formatClickhouseDate(cursor)}'`;
}
sb.limit = take;
sb.offset = Math.max(0, (cursor ?? 0) * take);
sb.where.projectId = `project_id = ${escape(projectId)}`;
const select = mergeDeepRight(
{
@@ -380,6 +401,7 @@ export async function getEventList({
name: true,
deviceId: true,
profileId: true,
sessionId: true,
projectId: true,
createdAt: true,
path: true,
@@ -607,3 +629,320 @@ export async function getTopPages({
return res;
}
export interface IEventServiceGetList {
projectId: string;
profileId?: string;
cursor?: Date;
filters?: IChartEventFilter[];
}
class EventService {
constructor(private client: typeof ch) {}
query<T>({
projectId,
profileId,
where,
select,
limit,
orderBy,
}: {
projectId: string;
profileId?: string;
where?: {
profile?: (query: Query<T>) => void;
event?: (query: Query<T>) => void;
session?: (query: Query<T>) => void;
};
select: {
profile?: Partial<SelectHelper<IServiceProfile>>;
event: Partial<SelectHelper<IServiceEvent>>;
};
limit?: number;
orderBy?: keyof IClickhouseEvent;
}) {
const events = clix(this.client)
.select<
Partial<IClickhouseEvent> & {
// profile
profileId: string;
profile_firstName: string;
profile_lastName: string;
profile_avatar: string;
profile_isExternal: boolean;
profile_createdAt: string;
}
>([
select.event.id && 'e.id as id',
select.event.deviceId && 'e.device_id as device_id',
select.event.name && 'e.name as name',
select.event.path && 'e.path as path',
select.event.duration && 'e.duration as duration',
select.event.country && 'e.country as country',
select.event.city && 'e.city as city',
select.event.os && 'e.os as os',
select.event.browser && 'e.browser as browser',
select.event.createdAt && 'e.created_at as created_at',
select.event.projectId && 'e.project_id as project_id',
'e.session_id as session_id',
'e.profile_id as profile_id',
])
.from('events e')
.where('project_id', '=', projectId)
.when(!!where?.event, where?.event)
// Do not limit if profileId, we will limit later since we need the "correct" profileId
.when(!!limit && !profileId, (q) => q.limit(limit!))
.orderBy('toDate(created_at)', 'DESC')
.orderBy('created_at', 'DESC');
const sessions = clix(this.client)
.select(['id as session_id', 'profile_id'])
.from('sessions')
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.when(!!where?.session, where?.session)
.when(!!profileId, (q) => q.where('profile_id', '=', profileId));
const profiles = clix(this.client)
.select([
'id',
'any(created_at) as created_at',
`any(nullIf(first_name, '')) as first_name`,
`any(nullIf(last_name, '')) as last_name`,
`any(nullIf(email, '')) as email`,
`any(nullIf(avatar, '')) as avatar`,
'last_value(is_external) as is_external',
])
.from('profiles')
.where('project_id', '=', projectId)
.where(
'id',
'IN',
clix.exp(
clix(this.client)
.select(['profile_id'])
.from(
clix.exp(
clix(this.client)
.select(['profile_id'])
.from('cte_sessions')
.union(
clix(this.client).select(['profile_id']).from('cte_events'),
),
),
)
.groupBy(['profile_id']),
),
)
.groupBy(['id', 'project_id'])
.when(!!where?.profile, where?.profile);
return clix(this.client)
.with('cte_events', events)
.with('cte_sessions', sessions)
.with('cte_profiles', profiles)
.select<
Partial<IClickhouseEvent> & {
// profile
profileId: string;
profile_firstName: string;
profile_lastName: string;
profile_avatar: string;
profile_isExternal: boolean;
profile_createdAt: string;
}
>([
select.event.id && 'e.id as id',
select.event.deviceId && 'e.device_id as device_id',
select.event.name && 'e.name as name',
select.event.path && 'e.path as path',
select.event.duration && 'e.duration as duration',
select.event.country && 'e.country as country',
select.event.city && 'e.city as city',
select.event.os && 'e.os as os',
select.event.browser && 'e.browser as browser',
select.event.createdAt && 'e.created_at as created_at',
select.event.projectId && 'e.project_id as project_id',
select.event.sessionId && 'e.session_id as session_id',
select.event.profileId && 'e.profile_id as event_profile_id',
// Profile
select.profile?.id && 'p.id as profile_id',
select.profile?.firstName && 'p.first_name as profile_first_name',
select.profile?.lastName && 'p.last_name as profile_last_name',
select.profile?.avatar && 'p.avatar as profile_avatar',
select.profile?.isExternal && 'p.is_external as profile_is_external',
select.profile?.createdAt && 'p.created_at as profile_created_at',
select.profile?.email && 'p.email as profile_email',
select.profile?.properties && 'p.properties as profile_properties',
])
.from('cte_events e')
.leftJoin('cte_sessions s', 'e.session_id = s.session_id')
.leftJoin(
'cte_profiles p',
's.profile_id = p.id AND p.is_external = true',
)
.when(!!profileId, (q) => {
q.where('s.profile_id', '=', profileId);
q.limit(limit!);
});
}
transformFromQuery(res: any[]) {
return res
.map((item) => {
return Object.entries(item).reduce(
(acc, [prop, val]) => {
if (prop === 'event_profile_id' && val) {
if (!item.profile_id) {
return assocPath(['profile', 'id'], val, acc);
}
}
if (
prop.startsWith('profile_') &&
!path(['profile', prop.replace('profile_', '')], acc)
) {
return assocPath(
['profile', prop.replace('profile_', '')],
val,
acc,
);
}
return assocPath([prop], val, acc);
},
{
profile: {},
} as IClickhouseEvent,
);
})
.map(transformEvent);
}
async getById({
projectId,
id,
createdAt,
}: {
projectId: string;
id: string;
createdAt?: Date;
}) {
return clix(this.client)
.select<IClickhouseEvent>(['*'])
.from('events')
.where('project_id', '=', projectId)
.when(!!createdAt, (q) => {
if (createdAt) {
q.where('created_at', 'BETWEEN', [
new Date(createdAt.getTime() - 1000),
new Date(createdAt.getTime() + 1000),
]);
}
})
.where('id', '=', id)
.limit(1)
.execute()
.then((res) => {
if (!res[0]) {
return null;
}
return transformEvent(res[0]);
});
}
async getList({
projectId,
profileId,
cursor,
filters,
limit = 50,
startDate,
endDate,
}: IEventServiceGetList & {
limit?: number;
startDate?: Date;
endDate?: Date;
}) {
const date = cursor || new Date();
const query = this.query({
projectId,
profileId,
limit,
orderBy: 'created_at',
select: {
event: {
deviceId: true,
profileId: true,
id: true,
name: true,
createdAt: true,
duration: true,
country: true,
city: true,
os: true,
browser: true,
path: true,
sessionId: true,
},
profile: {
id: true,
firstName: true,
lastName: true,
avatar: true,
isExternal: true,
},
},
where: {
event: (q) => {
if (startDate && endDate) {
q.where('created_at', 'BETWEEN', [
startDate ?? new Date(date.getTime() - 1000 * 60 * 60 * 24 * 3.5),
cursor ?? endDate,
]);
} else {
q.where('created_at', '<', date);
}
if (filters) {
q.rawWhere(
Object.values(getEventFiltersWhereClause(filters)).join(' AND '),
);
}
},
session: (q) => {
if (startDate && endDate) {
q.where('created_at', 'BETWEEN', [
startDate ?? new Date(date.getTime() - 1000 * 60 * 60 * 24 * 3.5),
endDate ?? date,
]);
} else {
q.where('created_at', '<', date);
}
},
},
})
.orderBy('toDate(created_at)', 'DESC')
.orderBy('created_at', 'DESC');
const results = await query.execute();
// Current page items (middle chunk)
const items = results.slice(0, limit);
// Check if there's a next page
const hasNext = results.length >= limit;
return {
items: this.transformFromQuery(items).map((item) => ({
...item,
projectId: projectId,
})),
meta: {
next: hasNext ? last(items)?.created_at : null,
},
};
}
}
export const eventService = new EventService(ch);

639
packages/db/src/services/overview.service.ts Normal file
View File

@@ -0,0 +1,639 @@
import { average, sum } from '@openpanel/common';
import { getCache } from '@openpanel/redis';
import { type IChartEventFilter, zTimeInterval } from '@openpanel/validation';
import { omit } from 'ramda';
import { z } from 'zod';
import { TABLE_NAMES, ch } from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
import { getEventFiltersWhereClause } from './chart.service';
export const zGetMetricsInput = z.object({
projectId: z.string(),
filters: z.array(z.any()),
startDate: z.string(),
endDate: z.string(),
interval: zTimeInterval,
});
export type IGetMetricsInput = z.infer<typeof zGetMetricsInput>;
export const zGetTopPagesInput = z.object({
projectId: z.string(),
filters: z.array(z.any()),
startDate: z.string(),
endDate: z.string(),
interval: zTimeInterval,
cursor: z.number().optional(),
limit: z.number().optional(),
});
export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput>;
export const zGetTopEntryExitInput = z.object({
projectId: z.string(),
filters: z.array(z.any()),
startDate: z.string(),
endDate: z.string(),
interval: zTimeInterval,
mode: z.enum(['entry', 'exit']),
cursor: z.number().optional(),
limit: z.number().optional(),
});
export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput>;
export const zGetTopGenericInput = z.object({
projectId: z.string(),
filters: z.array(z.any()),
startDate: z.string(),
endDate: z.string(),
interval: zTimeInterval,
column: z.enum([
// Referrers
'referrer',
'referrer_name',
'referrer_type',
'utm_source',
'utm_medium',
'utm_campaign',
'utm_term',
'utm_content',
// Geo
'region',
'country',
'city',
// Device
'device',
'brand',
'model',
'browser',
'browser_version',
'os',
'os_version',
]),
cursor: z.number().optional(),
limit: z.number().optional(),
});
export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput>;
export class OverviewService {
private pendingQueries: Map<string, Promise<number | null>> = new Map();
constructor(private client: typeof ch) {}
isPageFilter(filters: IChartEventFilter[]) {
return filters.some((filter) => filter.name === 'path' && filter.value);
}
getTotalSessions({
projectId,
startDate,
endDate,
filters,
}: {
projectId: string;
startDate: string;
endDate: string;
filters: IChartEventFilter[];
}) {
const where = this.getRawWhereClause('sessions', filters);
const key = `total_sessions_${projectId}_${startDate}_${endDate}_${JSON.stringify(filters)}`;
// Check if there's already a pending query for this key
const pendingQuery = this.pendingQueries.get(key);
if (pendingQuery) {
return pendingQuery.then((res) => res ?? 0);
}
// Create new query promise and store it
const queryPromise = getCache(key, 15, async () => {
try {
const result = await clix(this.client)
.select<{
total_sessions: number;
}>(['sum(sign) as total_sessions'])
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.rawWhere(where)
.having('sum(sign)', '>', 0)
.execute();
return result?.[0]?.total_sessions ?? 0;
} catch (error) {
return 0;
}
});
this.pendingQueries.set(key, queryPromise);
return queryPromise;
}
getMetrics({
projectId,
filters,
startDate,
endDate,
interval,
}: IGetMetricsInput): Promise<{
metrics: {
bounce_rate: number;
unique_visitors: number;
total_sessions: number;
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
};
series: {
date: string;
bounce_rate: number;
unique_visitors: number;
total_sessions: number;
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
}[];
}> {
const where = this.getRawWhereClause('sessions', filters);
if (this.isPageFilter(filters)) {
// Session aggregation with bounce rates
const sessionAggQuery = clix(this.client)
.select([
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
])
.from(TABLE_NAMES.sessions, true)
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.rawWhere(where)
.groupBy(['date'])
.rollup()
.orderBy('date', 'ASC');
// Overall unique visitors
const overallUniqueVisitorsQuery = clix(this.client)
.select([
'uniq(profile_id) AS unique_visitors',
'uniq(session_id) AS total_sessions',
])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.rawWhere(this.getRawWhereClause('events', filters));
return clix(this.client)
.with('session_agg', sessionAggQuery)
.with(
'overall_bounce_rate',
clix(this.client)
.select(['bounce_rate'])
.from('session_agg')
.where('date', '=', clix.exp("'1970-01-01 00:00:00'")),
)
.with(
'daily_stats',
clix(this.client)
.select(['date', 'bounce_rate'])
.from('session_agg')
.where('date', '!=', clix.exp("'1970-01-01 00:00:00'")),
)
.with('overall_unique_visitors', overallUniqueVisitorsQuery)
.select<{
date: string;
bounce_rate: number;
unique_visitors: number;
total_sessions: number;
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
overall_unique_visitors: number;
overall_total_sessions: number;
overall_bounce_rate: number;
}>([
`${clix.toStartOfInterval('e.created_at', interval, startDate)} AS date`,
'ds.bounce_rate as bounce_rate',
'uniq(e.profile_id) AS unique_visitors',
'uniq(e.session_id) AS total_sessions',
'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
'count(*) AS total_screen_views',
'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
'(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
'(SELECT total_sessions FROM overall_unique_visitors) AS overall_total_sessions',
'(SELECT bounce_rate FROM overall_bounce_rate) AS overall_bounce_rate',
])
.from(`${TABLE_NAMES.events} AS e`)
.leftJoin(
'daily_stats AS ds',
`${clix.toStartOfInterval('e.created_at', interval, startDate)} = ds.date`,
)
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.rawWhere(this.getRawWhereClause('events', filters))
.groupBy(['date', 'ds.bounce_rate'])
.orderBy('date', 'ASC')
.fill(
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
clix.toInterval('1', interval),
)
.transform({
date: (item) => new Date(item.date).toISOString(),
})
.execute()
.then((res) => {
const anyRowWithData = res.find(
(item) =>
item.overall_bounce_rate !== null ||
item.overall_total_sessions !== null ||
item.overall_unique_visitors !== null,
);
return {
metrics: {
bounce_rate: anyRowWithData?.overall_bounce_rate ?? 0,
unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0,
total_sessions: anyRowWithData?.overall_total_sessions ?? 0,
avg_session_duration: average(
res.map((item) => item.avg_session_duration),
),
total_screen_views: sum(
res.map((item) => item.total_screen_views),
),
views_per_session: average(
res.map((item) => item.views_per_session),
),
},
series: res.map(
omit([
'overall_bounce_rate',
'overall_unique_visitors',
'overall_total_sessions',
]),
),
};
});
}
const query = clix(this.client)
.select<{
date: string;
bounce_rate: number;
unique_visitors: number;
total_sessions: number;
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
}>([
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
'uniqIf(profile_id, sign > 0) AS unique_visitors',
'sum(sign) AS total_sessions',
'round(avgIf(duration, duration > 0 AND sign > 0), 2) / 1000 AS _avg_session_duration',
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
'sum(sign * screen_view_count) AS total_screen_views',
'round(sum(sign * screen_view_count) * 1.0 / sum(sign), 2) AS views_per_session',
])
.from('sessions')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.where('project_id', '=', projectId)
.rawWhere(where)
.groupBy(['date'])
.having('sum(sign)', '>', 0)
.rollup()
.orderBy('date', 'ASC')
.fill(
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
clix.toInterval('1', interval),
)
.transform({
date: (item) => new Date(item.date).toISOString(),
});
return query.execute().then((res) => {
// First row is the rollup row containing the total values
return {
metrics: {
bounce_rate: res[0]?.bounce_rate ?? 0,
unique_visitors: res[0]?.unique_visitors ?? 0,
total_sessions: res[0]?.total_sessions ?? 0,
avg_session_duration: res[0]?.avg_session_duration ?? 0,
total_screen_views: res[0]?.total_screen_views ?? 0,
views_per_session: res[0]?.views_per_session ?? 0,
},
series: res
.slice(1)
.map(omit(['overall_bounce_rate', 'overall_unique_visitors'])),
};
});
}
getRawWhereClause(type: 'events' | 'sessions', filters: IChartEventFilter[]) {
const where = getEventFiltersWhereClause(
filters.map((item) => {
if (type === 'sessions') {
if (item.name === 'path') {
return { ...item, name: 'entry_path' };
}
if (item.name === 'origin') {
return { ...item, name: 'entry_origin' };
}
if (item.name.startsWith('properties.__query.utm_')) {
return {
...item,
name: item.name.replace('properties.__query.utm_', 'utm_'),
};
}
return item;
}
return item;
}),
// .filter((item) => {
// if (this.isPageFilter(filters) && type === 'sessions') {
// return item.name !== 'entry_path' && item.name !== 'entry_origin';
// }
// return true;
// }),
);
return Object.values(where).join(' AND ');
}
async getTopPages({
projectId,
filters,
startDate,
endDate,
cursor = 1,
limit = 10,
}: IGetTopPagesInput) {
const pageStatsQuery = clix(this.client)
.select([
'origin',
'path',
'uniq(session_id) as count',
'round(avg(duration)/1000, 2) as avg_duration',
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('path', '!=', '')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.groupBy(['origin', 'path'])
.orderBy('count', 'DESC')
.limit(limit)
.offset((cursor - 1) * limit);
const bounceStatsQuery = clix(this.client)
.select([
'entry_path',
'entry_origin',
'coalesce(round(countIf(is_bounce = 1 AND sign = 1) * 100.0 / countIf(sign = 1), 2), 0) as bounce_rate',
])
.from(TABLE_NAMES.sessions, true)
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.groupBy(['entry_path', 'entry_origin']);
pageStatsQuery.rawWhere(this.getRawWhereClause('events', filters));
bounceStatsQuery.rawWhere(this.getRawWhereClause('sessions', filters));
const mainQuery = clix(this.client)
.with('page_stats', pageStatsQuery)
.with('bounce_stats', bounceStatsQuery)
.select<{
origin: string;
path: string;
avg_duration: number;
bounce_rate: number;
sessions: number;
}>([
'p.origin',
'p.path',
'p.avg_duration',
'p.count as sessions',
'b.bounce_rate',
])
.from('page_stats p', false)
.leftJoin(
'bounce_stats b',
'p.path = b.entry_path AND p.origin = b.entry_origin',
)
.orderBy('sessions', 'DESC')
.limit(limit);
const totalSessions = await this.getTotalSessions({
projectId,
startDate,
endDate,
filters,
});
return mainQuery.execute();
}
async getTopEntryExit({
projectId,
filters,
startDate,
endDate,
mode,
cursor = 1,
limit = 10,
}: IGetTopEntryExitInput) {
const where = this.getRawWhereClause('sessions', filters);
const distinctSessionQuery = this.getDistinctSessions({
projectId,
filters,
startDate,
endDate,
});
const offset = (cursor - 1) * limit;
const query = clix(this.client)
.select<{
origin: string;
path: string;
avg_duration: number;
bounce_rate: number;
sessions: number;
}>([
`${mode}_origin AS origin`,
`${mode}_path AS path`,
'round(avg(duration * sign)/1000, 2) as avg_duration',
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
'sum(sign) as sessions',
])
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.rawWhere(where)
.groupBy([`${mode}_origin`, `${mode}_path`])
.having('sum(sign)', '>', 0)
.orderBy('sessions', 'DESC')
.limit(limit)
.offset(offset);
let mainQuery = query;
if (this.isPageFilter(filters)) {
mainQuery = clix(this.client)
.with('distinct_sessions', distinctSessionQuery)
.merge(query)
.where(
'id',
'IN',
clix.exp('(SELECT session_id FROM distinct_sessions)'),
);
}
const totalSessions = await this.getTotalSessions({
projectId,
startDate,
endDate,
filters,
});
return mainQuery.execute();
}
private getDistinctSessions({
projectId,
filters,
startDate,
endDate,
}: {
projectId: string;
filters: IChartEventFilter[];
startDate: string;
endDate: string;
}) {
return clix(this.client)
.select(['DISTINCT session_id'])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.rawWhere(this.getRawWhereClause('events', filters));
}
async getTopGeneric({
projectId,
filters,
startDate,
endDate,
column,
cursor = 1,
limit = 10,
}: IGetTopGenericInput) {
const distinctSessionQuery = this.getDistinctSessions({
projectId,
filters,
startDate,
endDate,
});
const prefixColumn = (() => {
switch (column) {
case 'region':
return 'country';
case 'city':
return 'country';
case 'browser_version':
return 'browser';
case 'os_version':
return 'os';
}
return null;
})();
const offset = (cursor - 1) * limit;
const query = clix(this.client)
.select<{
prefix?: string;
name: string;
sessions: number;
bounce_rate: number;
avg_session_duration: number;
}>([
prefixColumn && `${prefixColumn} as prefix`,
`nullIf(${column}, '') as name`,
'sum(sign) as sessions',
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) AS bounce_rate',
'round(avgIf(duration, duration > 0 AND sign > 0), 2)/1000 AS avg_session_duration',
])
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
])
.groupBy([prefixColumn, column].filter(Boolean))
.having('sum(sign)', '>', 0)
.orderBy('sessions', 'DESC')
.limit(limit)
.offset(offset);
let mainQuery = query;
if (this.isPageFilter(filters)) {
mainQuery = clix(this.client)
.with('distinct_sessions', distinctSessionQuery)
.merge(query)
.where(
'id',
'IN',
clix.exp('(SELECT session_id FROM distinct_sessions)'),
);
} else {
mainQuery.rawWhere(this.getRawWhereClause('sessions', filters));
}
const [res, totalSessions] = await Promise.all([
mainQuery.execute(),
this.getTotalSessions({
projectId,
startDate,
endDate,
filters,
}),
]);
return res;
}
}
export const overviewService = new OverviewService(ch);

28
packages/db/src/services/profile.service.ts
View File

@@ -49,7 +49,17 @@ export async function getProfileById(id: string, projectId: string) {
}
const [profile] = await chQuery<IClickhouseProfile>(
`SELECT * FROM ${TABLE_NAMES.profiles} WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`,
`SELECT
id,
project_id,
last_value(nullIf(first_name, '')) as first_name,
last_value(nullIf(last_name, '')) as last_name,
last_value(nullIf(email, '')) as email,
last_value(nullIf(avatar, '')) as avatar,
last_value(is_external) as is_external,
last_value(properties) as properties,
last_value(created_at) as created_at
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
);
if (!profile) {
@@ -59,7 +69,7 @@ export async function getProfileById(id: string, projectId: string) {
return transformProfile(profile);
}
export const getProfileByIdCached = cacheable(getProfileById, 60 * 30);
export const getProfileByIdCached = getProfileById; //cacheable(getProfileById, 60 * 30);
interface GetProfileListOptions {
projectId: string;
@@ -77,11 +87,21 @@ export async function getProfiles(ids: string[], projectId: string) {
}
const data = await chQuery<IClickhouseProfile>(
`SELECT id, first_name, last_name, email, avatar, is_external, properties, created_at
FROM ${TABLE_NAMES.profiles} FINAL
`SELECT
id,
project_id,
any(nullIf(first_name, '')) as first_name,
any(nullIf(last_name, '')) as last_name,
any(nullIf(email, '')) as email,
any(nullIf(avatar, '')) as avatar,
last_value(is_external) as is_external,
any(properties) as properties,
any(created_at) as created_at
FROM ${TABLE_NAMES.profiles}
WHERE
project_id = ${escape(projectId)} AND
id IN (${filteredIds.map((id) => escape(id)).join(',')})
GROUP BY id, project_id
`,
);

41
packages/db/src/services/session.service.ts Normal file
View File

@@ -0,0 +1,41 @@
export type IClickhouseSession = {
id: string;
profile_id: string;
event_count: number;
screen_view_count: number;
screen_views: string[];
entry_path: string;
entry_origin: string;
exit_path: string;
exit_origin: string;
created_at: string;
ended_at: string;
referrer: string;
referrer_name: string;
referrer_type: string;
os: string;
os_version: string;
browser: string;
browser_version: string;
device: string;
brand: string;
model: string;
country: string;
region: string;
city: string;
longitude: number | null;
latitude: number | null;
is_bounce: boolean;
project_id: string;
device_id: string;
duration: number;
utm_medium: string;
utm_source: string;
utm_campaign: string;
utm_content: string;
utm_term: string;
revenue: number;
sign: 1 | 0;
version: number;
properties: Record<string, string>;
};