zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: dashboard v2, esm, upgrades (#211)

Browse Source 
* esm

* wip

* wip

* wip

* wip

* wip

* wip

* subscription notice

* wip

* wip

* wip

* fix envs

* fix: update docker build

* fix

* esm/types

* delete dashboard :D

* add patches to dockerfiles

* update packages + catalogs + ts

* wip

* remove native libs

* ts

* improvements

* fix redirects and fetching session

* try fix favicon

* fixes

* fix

* order and resize reportds within a dashboard

* improvements

* wip

* added userjot to dashboard

* fix

* add op

* wip

* different cache key

* improve date picker

* fix table

* event details loading

* redo onboarding completely

* fix login

* fix

* fix

* extend session, billing and improve bars

* fix

* reduce price on 10M
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-16 12:27:44 +02:00
committed by GitHub
parent 436e81ecc9
commit 81a7e5d62e
741 changed files with 32695 additions and 16996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
packages/db/src/services/chart.service.ts
View File

@@ -1,4 +1,4 @@
import { escape } from 'sqlstring';
import sqlstring from 'sqlstring';
import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common';
import type {
@@ -45,7 +45,7 @@ export function getSelectPropertyKey(property: string) {
if (!match) return property;
if (property.includes('*')) {
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${escape(
return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${sqlstring.escape(
transformPropertyKey(property),
)})))`;
}
@@ -76,11 +76,11 @@ export function getChartSql({
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.projectId = `project_id = ${escape(projectId)}`;
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (event.name !== '*') {
sb.select.label_0 = `${escape(event.name)} as label_0`;
sb.where.eventName = `name = ${escape(event.name)}`;
sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
} else {
sb.select.label_0 = `'*' as label_0`;
}
@@ -99,7 +99,7 @@ export function getChartSql({
first_name as "profile.first_name",
last_name as "profile.last_name",
properties as "profile.properties"
FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`;
FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
}
sb.select.count = 'count(*) as count';
@@ -251,14 +251,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
case 'is': {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x = ${escape(String(val).trim())}`)
.map((val) => `x = ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else {
if (value.length === 1) {
where[id] = `${whereFrom} = ${escape(String(value[0]).trim())}`;
where[id] =
`${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} IN (${value
.map((val) => escape(String(val).trim()))
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
}
@@ -267,14 +268,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
case 'isNot': {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x != ${escape(String(val).trim())}`)
.map((val) => `x != ${sqlstring.escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else {
if (value.length === 1) {
where[id] = `${whereFrom} != ${escape(String(value[0]).trim())}`;
where[id] =
`${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${whereFrom} NOT IN (${value
.map((val) => escape(String(val).trim()))
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
}
@@ -283,13 +285,16 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
case 'contains': {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x LIKE ${escape(`%${String(val).trim()}%`)}`)
.map(
(val) =>
`x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${escape(`%${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
)
.join(' OR ')})`;
}
@@ -298,13 +303,16 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
case 'doesNotContain': {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x NOT LIKE ${escape(`%${String(val).trim()}%`)}`)
.map(
(val) =>
`x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} NOT LIKE ${escape(`%${String(val).trim()}%`)}`,
`${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
)
.join(' OR ')})`;
}
@@ -313,13 +321,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
case 'startsWith': {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x LIKE ${escape(`${String(val).trim()}%`)}`)
.map(
(val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${escape(`${String(val).trim()}%`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
)
.join(' OR ')})`;
}
@@ -328,13 +338,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
case 'endsWith': {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x LIKE ${escape(`%${String(val).trim()}`)}`)
.map(
(val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) =>
`${whereFrom} LIKE ${escape(`%${String(val).trim()}`)}`,
`${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
)
.join(' OR ')})`;
}
@@ -343,12 +355,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
case 'regex': {
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `match(x, ${escape(String(val).trim())})`)
.map((val) => `match(x, ${sqlstring.escape(String(val).trim())})`)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `(${value
.map(
(val) => `match(${whereFrom}, ${escape(String(val).trim())})`,
(val) =>
`match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`,
)
.join(' OR ')})`;
}
@@ -376,10 +389,11 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
switch (operator) {
case 'is': {
if (value.length === 1) {
where[id] = `${name} = ${escape(String(value[0]).trim())}`;
where[id] =
`${name} = ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${name} IN (${value
.map((val) => escape(String(val).trim()))
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
@@ -394,37 +408,48 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
}
case 'isNot': {
if (value.length === 1) {
where[id] = `${name} != ${escape(String(value[0]).trim())}`;
where[id] =
`${name} != ${sqlstring.escape(String(value[0]).trim())}`;
} else {
where[id] = `${name} NOT IN (${value
.map((val) => escape(String(val).trim()))
.map((val) => sqlstring.escape(String(val).trim()))
.join(', ')})`;
}
break;
}
case 'contains': {
where[id] = `(${value
.map((val) => `${name} LIKE ${escape(`%${String(val).trim()}%`)}`)
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
)
.join(' OR ')})`;
break;
}
case 'doesNotContain': {
where[id] = `(${value
.map(
(val) => `${name} NOT LIKE ${escape(`%${String(val).trim()}%`)}`,
(val) =>
`${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
)
.join(' OR ')})`;
break;
}
case 'startsWith': {
where[id] = `(${value
.map((val) => `${name} LIKE ${escape(`${String(val).trim()}%`)}`)
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
)
.join(' OR ')})`;
break;
}
case 'endsWith': {
where[id] = `(${value
.map((val) => `${name} LIKE ${escape(`%${String(val).trim()}`)}`)
.map(
(val) =>
`${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
)
.join(' OR ')})`;
break;
}
@@ -432,7 +457,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
where[id] = `(${value
.map(
(val) =>
`match(${name}, ${escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
`match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
)
.join(' OR ')})`;
break;

155
packages/db/src/services/event.service.ts
View File

@@ -1,5 +1,5 @@
import { path, assocPath, last, mergeDeepRight } from 'ramda';
import { escape } from 'sqlstring';
import sqlstring from 'sqlstring';
import { v4 as uuid } from 'uuid';
import { DateTime, toDots } from '@openpanel/common';
@@ -17,10 +17,17 @@ import {
import { type Query, clix } from '../clickhouse/query-builder';
import type { EventMeta, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
import { createSqlBuilder } from '../sql-builder';
import { type SqlBuilderObject, createSqlBuilder } from '../sql-builder';
import { getEventFiltersWhereClause } from './chart.service';
import { getOrganizationByProjectIdCached } from './organization.service';
import type { IServiceProfile, IServiceUpsertProfile } from './profile.service';
import { getProfileById, getProfiles, upsertProfile } from './profile.service';
import {
getProfileById,
getProfileByIdCached,
getProfiles,
getProfilesCached,
upsertProfile,
} from './profile.service';
export type IImportedEvent = Omit<
IClickhouseEvent,
@@ -258,7 +265,7 @@ export async function getEvents(
const ids = events
.filter((e) => e.device_id !== e.profile_id)
.map((e) => e.profile_id);
const profiles = await getProfiles(ids, projectId);
const profiles = await getProfilesCached(ids, projectId);
const map = new Map<string, IServiceProfile>();
for (const profile of profiles) {
@@ -266,7 +273,17 @@ export async function getEvents(
}
for (const event of events) {
event.profile = map.get(event.profile_id);
event.profile = map.get(event.profile_id) ?? {
id: event.profile_id,
email: '',
avatar: '',
firstName: '',
lastName: '',
createdAt: new Date(),
projectId,
isExternal: false,
properties: {},
};
}
}
@@ -365,6 +382,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
export interface GetEventListOptions {
projectId: string;
profileId?: string;
sessionId?: string;
take: number;
cursor?: number | Date;
events?: string[] | null;
@@ -372,29 +390,46 @@ export interface GetEventListOptions {
startDate?: Date;
endDate?: Date;
select?: SelectHelper<IServiceEvent>;
custom?: (sb: SqlBuilderObject) => void;
}
export async function getEventList({
cursor,
take,
projectId,
profileId,
events,
filters,
startDate,
endDate,
select: incomingSelect,
}: GetEventListOptions) {
export async function getEventList(options: GetEventListOptions) {
const {
cursor,
take,
projectId,
profileId,
sessionId,
events,
filters,
startDate,
endDate,
select: incomingSelect,
custom,
} = options;
const { sb, getSql, join } = createSqlBuilder();
const organization = await getOrganizationByProjectIdCached(projectId);
// This will speed up the query quite a lot for big organizations
const dateIntervalInDays =
organization?.subscriptionPeriodEventsLimit &&
organization?.subscriptionPeriodEventsLimit > 1_000_000
? 1
: 7;
if (typeof cursor === 'number') {
sb.offset = Math.max(0, (cursor ?? 0) * take);
} else if (cursor instanceof Date) {
sb.where.cursor = `created_at <= '${formatClickhouseDate(cursor)}'`;
sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(cursor))}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
sb.where.cursor = `created_at <= ${sqlstring.escape(formatClickhouseDate(cursor))}`;
}
if (!cursor) {
sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
}
sb.limit = take;
sb.where.projectId = `project_id = ${escape(projectId)}`;
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
const select = mergeDeepRight(
{
id: true,
@@ -503,7 +538,11 @@ export async function getEventList({
}
if (profileId) {
sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`;
sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND device_id != '' AND profile_id = ${sqlstring.escape(profileId)} group by did) OR profile_id = ${sqlstring.escape(profileId)})`;
}
if (sessionId) {
sb.where.sessionId = `session_id = ${sqlstring.escape(sessionId)}`;
}
if (startDate && endDate) {
@@ -527,10 +566,29 @@ export async function getEventList({
sb.orderBy.created_at =
'toDate(created_at) DESC, created_at DESC, profile_id DESC, name DESC';
return getEvents(getSql(), {
if (custom) {
custom(sb);
}
console.log('getSql()', getSql());
const data = await getEvents(getSql(), {
profile: select.profile ?? true,
meta: select.meta ?? true,
});
// If we dont get any events, try without the cursor window
if (data.length === 0 && sb.where.cursorWindow) {
return getEventList({
...options,
custom(sb) {
options.custom?.(sb);
delete sb.where.cursorWindow;
},
});
}
return data;
}
export const getEventsCountCached = cacheable(getEventsCount, 60 * 10);
@@ -543,9 +601,9 @@ export async function getEventsCount({
endDate,
}: Omit<GetEventListOptions, 'cursor' | 'take'>) {
const { sb, getSql, join } = createSqlBuilder();
sb.where.projectId = `project_id = ${escape(projectId)}`;
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (profileId) {
sb.where.profileId = `profile_id = ${escape(profileId)}`;
sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
}
if (startDate && endDate) {
@@ -614,7 +672,7 @@ export async function getTopPages({
SELECT path, count(*) as count, project_id, first_value(created_at) as first_seen, last_value(properties['__title']) as title, origin
FROM ${TABLE_NAMES.events}
WHERE name = 'screen_view'
AND project_id = ${escape(projectId)}
AND project_id = ${sqlstring.escape(projectId)}
AND created_at > now() - INTERVAL 30 DAY
${search ? `AND path ILIKE '%${search}%'` : ''}
GROUP BY path, project_id, origin
@@ -824,36 +882,43 @@ class EventService {
id: string;
createdAt?: Date;
}) {
const event = await clix(this.client)
.select<IClickhouseEvent>(['*'])
.from('events')
.where('project_id', '=', projectId)
.when(!!createdAt, (q) => {
if (createdAt) {
q.where('created_at', 'BETWEEN', [
new Date(createdAt.getTime() - 1000),
new Date(createdAt.getTime() + 1000),
]);
}
})
.where('id', '=', id)
.limit(1)
.execute()
.then((res) => {
if (!res[0]) {
return null;
}
const [event, metas] = await Promise.all([
clix(this.client)
.select<IClickhouseEvent>(['*'])
.from('events')
.where('project_id', '=', projectId)
.when(!!createdAt, (q) => {
if (createdAt) {
q.where('created_at', 'BETWEEN', [
new Date(createdAt.getTime() - 1000),
new Date(createdAt.getTime() + 1000),
]);
}
})
.where('id', '=', id)
.limit(1)
.execute()
.then((res) => {
if (!res[0]) {
return null;
}
return transformEvent(res[0]);
});
return transformEvent(res[0]);
}),
getEventMetasCached(projectId),
]);
if (event?.profileId) {
const profile = await getProfileById(event?.profileId, projectId);
const profile = await getProfileByIdCached(event?.profileId, projectId);
if (profile) {
event.profile = profile;
}
}
if (event) {
event.meta = metas.find((meta) => meta.name === event.name);
}
return event;
}

4
packages/db/src/services/funnel.service.ts
View File

@@ -1,7 +1,7 @@
import { ifNaN } from '@openpanel/common';
import type { IChartEvent, IChartInput } from '@openpanel/validation';
import { last, reverse } from 'ramda';
import { escape } from 'sqlstring';
import sqlstring from 'sqlstring';
import { ch } from '../clickhouse/client';
import { TABLE_NAMES } from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
@@ -24,7 +24,7 @@ export class FunnelService {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.name = `name = ${escape(event.name)}`;
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
}

24
packages/db/src/services/organization.service.ts
View File

@@ -1,6 +1,6 @@
import { DateTime } from '@openpanel/common';
import { cacheable } from '@openpanel/redis';
import { escape } from 'sqlstring';
import sqlstring from 'sqlstring';
import { chQuery, formatClickhouseDate } from '../clickhouse/client';
import type { Invite, Prisma, ProjectAccess, User } from '../prisma-client';
import { db } from '../prisma-client';
@@ -69,11 +69,14 @@ export async function getInvites(organizationId: string) {
where: {
organizationId,
},
orderBy: {
createdAt: 'desc',
},
});
}
export function getInviteById(inviteId: string) {
return db.invite.findUnique({
export async function getInviteById(inviteId: string) {
const res = await db.invite.findUnique({
where: {
id: inviteId,
},
@@ -86,6 +89,11 @@ export function getInviteById(inviteId: string) {
},
},
});
return {
...res,
isExpired: res?.expiresAt && res.expiresAt < new Date(),
};
}
export async function getMembers(organizationId: string) {
@@ -201,8 +209,8 @@ export async function getOrganizationBillingEventsCount(
const { sb, getSql } = createSqlBuilder();
sb.select.count = 'COUNT(*) AS count';
sb.where.projectIds = `project_id IN (${organization.projects.map((project) => escape(project.id)).join(',')})`;
sb.where.createdAt = `created_at BETWEEN ${escape(formatClickhouseDate(organization.subscriptionCurrentPeriodStart))} AND ${escape(formatClickhouseDate(organization.subscriptionCurrentPeriodEnd))}`;
sb.where.projectIds = `project_id IN (${organization.projects.map((project) => sqlstring.escape(project.id)).join(',')})`;
sb.where.createdAt = `created_at BETWEEN ${sqlstring.escape(formatClickhouseDate(organization.subscriptionCurrentPeriodStart))} AND ${sqlstring.escape(formatClickhouseDate(organization.subscriptionCurrentPeriodEnd))}`;
const res = await chQuery<{ count: number }>(getSql());
return res[0]?.count;
@@ -224,9 +232,9 @@ export async function getOrganizationBillingEventsCountSerie(
sb.select.count = 'COUNT(*) AS count';
sb.select.day = `toDate(toStartOf${interval.slice(0, 1).toUpperCase() + interval.slice(1)}(created_at)) AS ${interval}`;
sb.groupBy.day = interval;
sb.orderBy.day = `${interval} WITH FILL FROM toDate(${escape(formatClickhouseDate(startDate, true))}) TO toDate(${escape(formatClickhouseDate(endDate, true))}) STEP INTERVAL 1 ${interval.toUpperCase()}`;
sb.where.projectIds = `project_id IN (${organization.projects.map((project) => escape(project.id)).join(',')})`;
sb.where.createdAt = `${interval} BETWEEN ${escape(formatClickhouseDate(startDate, true))} AND ${escape(formatClickhouseDate(endDate, true))}`;
sb.orderBy.day = `${interval} WITH FILL FROM toDate(${sqlstring.escape(formatClickhouseDate(startDate, true))}) TO toDate(${sqlstring.escape(formatClickhouseDate(endDate, true))}) STEP INTERVAL 1 ${interval.toUpperCase()}`;
sb.where.projectIds = `project_id IN (${organization.projects.map((project) => sqlstring.escape(project.id)).join(',')})`;
sb.where.createdAt = `${interval} BETWEEN ${sqlstring.escape(formatClickhouseDate(startDate, true))} AND ${sqlstring.escape(formatClickhouseDate(endDate, true))}`;
const res = await chQuery<{ count: number; day: string }>(getSql());
return res;

85
packages/db/src/services/profile.service.ts
View File

@@ -1,5 +1,5 @@
import { omit, uniq } from 'ramda';
import { escape } from 'sqlstring';
import sqlstring from 'sqlstring';
import { strip, toObject } from '@openpanel/common';
import { cacheable } from '@openpanel/redis';
@@ -21,25 +21,69 @@ export type IProfileMetrics = {
sessions: number;
durationAvg: number;
durationP90: number;
totalEvents: number;
uniqueDaysActive: number;
bounceRate: number;
avgEventsPerSession: number;
conversionEvents: number;
avgTimeBetweenSessions: number;
};
export function getProfileMetrics(profileId: string, projectId: string) {
return chQuery<IProfileMetrics>(`
WITH lastSeen AS (
SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
firstSeen AS (
SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
screenViews AS (
SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
sessions AS (
SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
duration AS (
SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
SELECT
round(avg(duration) / 1000 / 60, 2) as durationAvg,
round(quantilesExactInclusive(0.9)(duration)[1] / 1000 / 60, 2) as durationP90
FROM ${TABLE_NAMES.events}
WHERE name = 'session_end' AND duration != 0 AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
totalEvents AS (
SELECT count(*) as totalEvents FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
uniqueDaysActive AS (
SELECT count(DISTINCT toDate(created_at)) as uniqueDaysActive FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
bounceRate AS (
SELECT round(avg(properties['__bounce'] = '1') * 100, 4) as bounceRate FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
avgEventsPerSession AS (
SELECT round((SELECT totalEvents FROM totalEvents) / nullIf((SELECT sessions FROM sessions), 0), 2) as avgEventsPerSession
),
conversionEvents AS (
SELECT count(*) as conversionEvents FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
),
avgTimeBetweenSessions AS (
SELECT
CASE
WHEN (SELECT sessions FROM sessions) <= 1 THEN 0
ELSE round(dateDiff('second', (SELECT firstSeen FROM firstSeen), (SELECT lastSeen FROM lastSeen)) / nullIf((SELECT sessions FROM sessions) - 1, 0), 1)
END as avgTimeBetweenSessions
)
SELECT lastSeen, firstSeen, screenViews, sessions, durationAvg, durationP90 FROM lastSeen, firstSeen, screenViews,sessions, duration
SELECT
(SELECT lastSeen FROM lastSeen) as lastSeen,
(SELECT firstSeen FROM firstSeen) as firstSeen,
(SELECT screenViews FROM screenViews) as screenViews,
(SELECT sessions FROM sessions) as sessions,
(SELECT durationAvg FROM duration) as durationAvg,
(SELECT durationP90 FROM duration) as durationP90,
(SELECT totalEvents FROM totalEvents) as totalEvents,
(SELECT uniqueDaysActive FROM uniqueDaysActive) as uniqueDaysActive,
(SELECT bounceRate FROM bounceRate) as bounceRate,
(SELECT avgEventsPerSession FROM avgEventsPerSession) as avgEventsPerSession,
(SELECT conversionEvents FROM conversionEvents) as conversionEvents,
(SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions
`).then((data) => data[0]!);
}
@@ -59,7 +103,7 @@ export async function getProfileById(id: string, projectId: string) {
last_value(is_external) as is_external,
last_value(properties) as properties,
last_value(created_at) as created_at
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
);
if (!profile) {
@@ -77,6 +121,7 @@ interface GetProfileListOptions {
cursor?: number;
filters?: IChartEventFilter[];
search?: string;
isExternal?: boolean;
}
export async function getProfiles(ids: string[], projectId: string) {
@@ -99,8 +144,8 @@ export async function getProfiles(ids: string[], projectId: string) {
any(created_at) as created_at
FROM ${TABLE_NAMES.profiles}
WHERE
project_id = ${escape(projectId)} AND
id IN (${filteredIds.map((id) => escape(id)).join(',')})
project_id = ${sqlstring.escape(projectId)} AND
id IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')})
GROUP BY id, project_id
`,
);
@@ -108,36 +153,48 @@ export async function getProfiles(ids: string[], projectId: string) {
return data.map(transformProfile);
}
export const getProfilesCached = cacheable(getProfiles, 60 * 5);
export async function getProfileList({
take,
cursor,
projectId,
filters,
search,
isExternal,
}: GetProfileListOptions) {
const { sb, getSql } = createSqlBuilder();
sb.from = `${TABLE_NAMES.profiles} FINAL`;
sb.select.all = '*';
sb.where.project_id = `project_id = ${escape(projectId)}`;
sb.where.project_id = `project_id = ${sqlstring.escape(projectId)}`;
sb.limit = take;
sb.offset = Math.max(0, (cursor ?? 0) * take);
sb.orderBy.created_at = 'created_at DESC';
if (search) {
sb.where.search = `(email ILIKE '%${search}%' OR first_name ILIKE '%${search}%' OR last_name ILIKE '%${search}%')`;
}
if (isExternal !== undefined) {
sb.where.external = `is_external = ${isExternal ? 'true' : 'false'}`;
}
const data = await chQuery<IClickhouseProfile>(getSql());
return data.map(transformProfile);
}
export async function getProfileListCount({
projectId,
filters,
isExternal,
search,
}: Omit<GetProfileListOptions, 'cursor' | 'take'>) {
const { sb, getSql } = createSqlBuilder();
sb.from = 'profiles';
sb.select.count = 'count(id) as count';
sb.where.project_id = `project_id = ${escape(projectId)}`;
sb.where.project_id = `project_id = ${sqlstring.escape(projectId)}`;
sb.groupBy.project_id = 'project_id';
if (search) {
sb.where.search = `(email ILIKE '%${search}%' OR first_name ILIKE '%${search}%' OR last_name ILIKE '%${search}%')`;
}
if (isExternal !== undefined) {
sb.where.external = `is_external = ${isExternal ? 'true' : 'false'}`;
}
const data = await chQuery<{ count: number }>(getSql());
return data[0]?.count ?? 0;
}

6
packages/db/src/services/project.service.ts
View File

@@ -1,5 +1,5 @@
import { cacheable } from '@openpanel/redis';
import { escape } from 'sqlstring';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, chQuery } from '../clickhouse/client';
import type { Prisma, Project } from '../prisma-client';
import { db } from '../prisma-client';
@@ -50,7 +50,7 @@ export async function getProjectsByOrganizationId(organizationId: string) {
organizationId,
},
orderBy: {
createdAt: 'desc',
eventsCount: 'desc',
},
});
}
@@ -104,7 +104,7 @@ export async function getProjects({
export const getProjectEventsCount = async (projectId: string) => {
const res = await chQuery<{ count: number }>(
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`,
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)}`,
);
return res[0]?.count;
};

16
packages/db/src/services/reference.service.ts
View File

@@ -16,19 +16,3 @@ export async function getReferenceById(id: string) {
return reference;
}
export async function getReferences({
where,
take,
skip,
}: {
where: Prisma.ReferenceWhereInput;
take?: number;
skip?: number;
}) {
return db.reference.findMany({
where,
take: take ?? 50,
skip,
});
}

13
packages/db/src/services/reports.service.ts
View File

@@ -14,7 +14,7 @@ import type {
} from '@openpanel/validation';
import { db } from '../prisma-client';
import type { Report as DbReport } from '../prisma-client';
import type { Report as DbReport, ReportLayout } from '../prisma-client';
export type IServiceReport = Awaited<ReturnType<typeof getReportById>>;
@@ -46,8 +46,8 @@ export function transformReportEvent(
}
export function transformReport(
report: DbReport,
): IChartProps & { id: string } {
report: DbReport & { layout?: ReportLayout | null },
): IChartProps & { id: string; layout?: ReportLayout | null } {
return {
id: report.id,
projectId: report.projectId,
@@ -68,6 +68,7 @@ export function transformReport(
criteria: (report.criteria as ICriteria) ?? undefined,
funnelGroup: report.funnelGroup ?? undefined,
funnelWindow: report.funnelWindow ?? undefined,
layout: report.layout ?? undefined,
};
}
@@ -77,6 +78,9 @@ export function getReportsByDashboardId(dashboardId: string) {
where: {
dashboardId,
},
include: {
layout: true,
},
})
.then((reports) => reports.map(transformReport));
}
@@ -86,6 +90,9 @@ export async function getReportById(id: string) {
where: {
id,
},
include: {
layout: true,
},
});
if (!report) {

14
packages/db/src/services/retention.service.ts
View File

@@ -1,4 +1,4 @@
import { escape } from 'sqlstring';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, chQuery } from '../clickhouse/client';
@@ -16,7 +16,7 @@ WITH
profile_id,
max(toWeek(created_at)) AS last_seen
FROM ${TABLE_NAMES.events}
WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)
WHERE (project_id = ${sqlstring.escape(projectId)}) AND (profile_id != device_id)
GROUP BY profile_id
),
n AS
@@ -25,7 +25,7 @@ WITH
profile_id,
min(toWeek(created_at)) AS first_seen
FROM ${TABLE_NAMES.events}
WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)
WHERE (project_id = ${sqlstring.escape(projectId)}) AND (profile_id != device_id)
GROUP BY profile_id
),
a AS
@@ -85,7 +85,7 @@ export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {
AND toStartOfWeek(events.created_at) = toStartOfWeek(future_events.created_at - toIntervalWeek(1))
AND future_events.profile_id != future_events.device_id
WHERE
project_id = ${escape(projectId)}
project_id = ${sqlstring.escape(projectId)}
AND events.profile_id != events.device_id
GROUP BY 1
ORDER BY date ASC`;
@@ -122,11 +122,11 @@ export function getRollingActiveUsers({
(
SELECT *
FROM ${TABLE_NAMES.dau_mv}
WHERE project_id = ${escape(projectId)}
WHERE project_id = ${sqlstring.escape(projectId)}
)
ARRAY JOIN range(${days}) AS n
)
WHERE project_id = ${escape(projectId)}
WHERE project_id = ${sqlstring.escape(projectId)}
GROUP BY date`;
return chQuery<IServiceRetentionRollingActiveUsers>(sql);
@@ -141,7 +141,7 @@ export function getRetentionLastSeenSeries({
max(created_at) AS last_active,
profile_id
FROM ${TABLE_NAMES.events}
WHERE (project_id = ${escape(projectId)}) AND (device_id != profile_id)
WHERE (project_id = ${sqlstring.escape(projectId)}) AND (device_id != profile_id)
GROUP BY profile_id
)
SELECT

297
packages/db/src/services/session.service.ts
View File

@@ -1,5 +1,18 @@
import { TABLE_NAMES, ch } from '../clickhouse/client';
import { cacheable } from '@openpanel/redis';
import type { IChartEventFilter } from '@openpanel/validation';
import { uniq } from 'ramda';
import sqlstring from 'sqlstring';
import {
TABLE_NAMES,
ch,
chQuery,
formatClickhouseDate,
} from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
import { createSqlBuilder } from '../sql-builder';
import { getEventFiltersWhereClause } from './chart.service';
import { getOrganizationByProjectIdCached } from './organization.service';
import { type IServiceProfile, getProfilesCached } from './profile.service';
export type IClickhouseSession = {
id: string;
@@ -43,17 +56,291 @@ export type IClickhouseSession = {
properties: Record<string, string>;
};
export interface IServiceSession {
id: string;
profileId: string;
eventCount: number;
screenViewCount: number;
entryPath: string;
entryOrigin: string;
exitPath: string;
exitOrigin: string;
createdAt: Date;
endedAt: Date;
referrer: string;
referrerName: string;
referrerType: string;
os: string;
osVersion: string;
browser: string;
browserVersion: string;
device: string;
brand: string;
model: string;
country: string;
region: string;
city: string;
longitude: number | null;
latitude: number | null;
isBounce: boolean;
projectId: string;
deviceId: string;
duration: number;
utmMedium: string;
utmSource: string;
utmCampaign: string;
utmContent: string;
utmTerm: string;
revenue: number;
properties: Record<string, string>;
profile?: IServiceProfile;
}
export interface GetSessionListOptions {
projectId: string;
profileId?: string;
take: number;
filters?: IChartEventFilter[];
startDate?: Date;
endDate?: Date;
search?: string;
cursor?: Cursor | null;
}
export function transformSession(session: IClickhouseSession): IServiceSession {
return {
id: session.id,
profileId: session.profile_id,
eventCount: session.event_count,
screenViewCount: session.screen_view_count,
entryPath: session.entry_path,
entryOrigin: session.entry_origin,
exitPath: session.exit_path,
exitOrigin: session.exit_origin,
createdAt: new Date(session.created_at),
endedAt: new Date(session.ended_at),
referrer: session.referrer,
referrerName: session.referrer_name,
referrerType: session.referrer_type,
os: session.os,
osVersion: session.os_version,
browser: session.browser,
browserVersion: session.browser_version,
device: session.device,
brand: session.brand,
model: session.model,
country: session.country,
region: session.region,
city: session.city,
longitude: session.longitude,
latitude: session.latitude,
isBounce: session.is_bounce,
projectId: session.project_id,
deviceId: session.device_id,
duration: session.duration,
utmMedium: session.utm_medium,
utmSource: session.utm_source,
utmCampaign: session.utm_campaign,
utmContent: session.utm_content,
utmTerm: session.utm_term,
revenue: session.revenue,
properties: session.properties,
profile: undefined,
};
}
type Direction = 'initial' | 'next' | 'prev';
type PageInfo = {
next?: Cursor; // use last row
};
type Cursor = {
createdAt: string; // ISO 8601 with ms
id: string;
};
export async function getSessionList({
cursor,
take,
projectId,
profileId,
filters,
startDate,
endDate,
search,
}: GetSessionListOptions) {
const { sb, getSql } = createSqlBuilder();
sb.from = `${TABLE_NAMES.sessions} FINAL`;
sb.limit = take;
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
if (startDate && endDate) {
sb.where.range = `created_at BETWEEN toDateTime('${formatClickhouseDate(startDate)}') AND toDateTime('${formatClickhouseDate(endDate)}')`;
}
if (profileId)
sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
if (search) {
const s = sqlstring.escape(`%${search}%`);
sb.where.search = `(entry_path ILIKE ${s} OR exit_path ILIKE ${s} OR referrer ILIKE ${s} OR referrer_name ILIKE ${s})`;
}
if (filters?.length) {
Object.assign(sb.where, getEventFiltersWhereClause(filters));
}
const organization = await getOrganizationByProjectIdCached(projectId);
// This will speed up the query quite a lot for big organizations
const dateIntervalInDays =
organization?.subscriptionPeriodEventsLimit &&
organization?.subscriptionPeriodEventsLimit > 1_000_000
? 1
: 7;
if (cursor) {
const cAt = sqlstring.escape(cursor.createdAt);
const cId = sqlstring.escape(cursor.id);
sb.where.cursor = `(created_at < toDateTime64(${cAt}, 3) OR (created_at = toDateTime64(${cAt}, 3) AND id < ${cId}))`;
sb.where.cursorWindow = `created_at >= toDateTime64(${cAt}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
sb.orderBy.created_at = 'toDate(created_at) DESC, created_at DESC, id DESC';
} else {
sb.orderBy.created_at = 'toDate(created_at) DESC, created_at DESC, id DESC';
sb.where.created_at = `created_at > now() - INTERVAL ${dateIntervalInDays} DAY`;
}
// ==== Select columns (as you had) ====
// sb.select.id = 'id'; sb.select.project_id = 'project_id'; ... etc.
const columns = [
'created_at',
'ended_at',
'id',
'profile_id',
'entry_path',
'exit_path',
'duration',
'is_bounce',
'referrer_name',
'referrer',
'country',
'city',
'os',
'browser',
'brand',
'model',
'device',
'screen_view_count',
'event_count',
'revenue',
];
columns.forEach((column) => {
sb.select[column] = column;
});
const sql = getSql();
const data = await chQuery<
IClickhouseSession & {
latestCreatedAt: string;
}
>(sql);
// Compute cursors from page edges
const last = data[take - 1];
const meta: PageInfo = {
next: last
? {
createdAt: last.created_at,
id: last.id,
}
: undefined,
};
// Profile hydration (unchanged)
const profileIds = data
.filter((e) => e.device_id !== e.profile_id)
.map((e) => e.profile_id);
const profiles = await getProfilesCached(profileIds, projectId);
const map = new Map<string, IServiceProfile>(profiles.map((p) => [p.id, p]));
const items = data.map(transformSession).map((item) => ({
...item,
profile: map.get(item.profileId) ?? {
id: item.profileId,
email: '',
avatar: '',
firstName: '',
lastName: '',
createdAt: new Date(),
projectId,
isExternal: false,
properties: {},
},
}));
return { items, meta };
}
export async function getSessionsCount({
projectId,
profileId,
filters,
startDate,
endDate,
search,
}: Omit<GetSessionListOptions, 'take' | 'cursor'>) {
const { sb, getSql } = createSqlBuilder();
sb.select.count = 'count(*) as count';
sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
sb.where.sign = 'sign = 1';
if (profileId) {
sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
}
if (startDate && endDate) {
sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
}
if (search) {
sb.where.search = `(entry_path ILIKE '%${search}%' OR exit_path ILIKE '%${search}%' OR referrer ILIKE '%${search}%' OR referrer_name ILIKE '%${search}%')`;
}
if (filters && filters.length > 0) {
const sessionFilters = getEventFiltersWhereClause(filters);
sb.where = {
...sb.where,
...sessionFilters,
};
}
sb.from = TABLE_NAMES.sessions;
const result = await chQuery<{ count: number }>(getSql());
return result[0]?.count ?? 0;
}
export const getSessionsCountCached = cacheable(getSessionsCount, 60 * 10);
class SessionService {
constructor(private client: typeof ch) {}
byId(sessionId: string, projectId: string) {
return clix(this.client)
async byId(sessionId: string, projectId: string) {
const result = await clix(this.client)
.select<IClickhouseSession>(['*'])
.from(TABLE_NAMES.sessions)
.where('id', '=', sessionId)
.where('project_id', '=', projectId)
.execute()
.then((res) => res[0]);
.where('sign', '=', 1)
.execute();
if (!result[0]) {
throw new Error('Session not found');
}
return transformSession(result[0]);
}
}