zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feature(dashboard,api): add timezone support

Browse Source 
* feat(dashboard): add support for today, yesterday etc (timezones)

* fix(db): escape js dates

* fix(dashboard): ensure we support default timezone

* final fixes

* remove complete series and add sql with fill instead
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-05-23 11:26:44 +02:00
committed by GitHub
parent 46bfeee131
commit 680727355b
48 changed files with 1817 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
packages/db/src/services/chart.service.ts
View File

@@ -1,19 +1,12 @@
import { escape } from 'sqlstring';
import {
getTimezoneFromDateString,
stripLeadingAndTrailingSlashes,
} from '@openpanel/common';
import { stripLeadingAndTrailingSlashes } from '@openpanel/common';
import type {
IChartEventFilter,
IGetChartDataInput,
} from '@openpanel/validation';
import {
TABLE_NAMES,
formatClickhouseDate,
toDate,
} from '../clickhouse/client';
import { TABLE_NAMES, formatClickhouseDate } from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
export function transformPropertyKey(property: string) {
@@ -61,9 +54,9 @@ export function getChartSql({
startDate,
endDate,
projectId,
chartType,
limit,
}: IGetChartDataInput) {
timezone,
}: IGetChartDataInput & { timezone: string }) {
const {
sb,
join,
@@ -73,6 +66,7 @@ export function getChartSql({
getSelect,
getOrderBy,
getGroupBy,
getFill,
} = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
@@ -99,34 +93,40 @@ export function getChartSql({
sb.select.count = 'count(*) as count';
switch (interval) {
case 'minute': {
sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfMinute(toDateTime('${startDate}')) TO toStartOfMinute(toDateTime('${endDate}')) STEP toIntervalMinute(1)`;
sb.select.date = 'toStartOfMinute(created_at) as date';
break;
}
case 'hour': {
sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfHour(toDateTime('${startDate}')) TO toStartOfHour(toDateTime('${endDate}')) STEP toIntervalHour(1)`;
sb.select.date = 'toStartOfHour(created_at) as date';
break;
}
case 'day': {
sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfDay(toDateTime('${startDate}')) TO toStartOfDay(toDateTime('${endDate}')) STEP toIntervalDay(1)`;
sb.select.date = 'toStartOfDay(created_at) as date';
break;
}
case 'week': {
sb.select.date = `toStartOfWeek(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfWeek(toDateTime('${startDate}'), 1, '${timezone}') TO toStartOfWeek(toDateTime('${endDate}'), 1, '${timezone}') STEP toIntervalWeek(1)`;
sb.select.date = `toStartOfWeek(created_at, 1, '${timezone}') as date`;
break;
}
case 'month': {
sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.fill = `FROM toStartOfMonth(toDateTime('${startDate}'), '${timezone}') TO toStartOfMonth(toDateTime('${endDate}'), '${timezone}') STEP toIntervalMonth(1)`;
sb.select.date = `toStartOfMonth(created_at, '${timezone}') as date`;
break;
}
}
sb.groupBy.date = 'date';
sb.orderBy.date = 'date ASC';
if (startDate) {
sb.where.startDate = `${toDate('created_at', interval)} >= ${toDate(formatClickhouseDate(startDate), interval)}`;
sb.where.startDate = `created_at >= toDateTime('${formatClickhouseDate(startDate)}')`;
}
if (endDate) {
sb.where.endDate = `${toDate('created_at', interval)} <= ${toDate(formatClickhouseDate(endDate), interval)}`;
sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;
}
if (breakdowns.length > 0 && limit) {
@@ -179,18 +179,14 @@ export function getChartSql({
ORDER BY profile_id, created_at DESC
) as subQuery`;
console.log(
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
);
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql);
return sql;
}
console.log(
`${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
);
return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;
console.log('CHART SQL', sql);
return sql;
}
export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {

22
packages/db/src/services/insights.service.ts
View File

@@ -1,5 +1,5 @@
import type { ClickHouseClient } from '@clickhouse/client';
import { Query, createQuery } from '../clickhouse/query-builder';
import { clix } from '../clickhouse/query-builder';
export interface Insight {
type: string;
@@ -73,7 +73,7 @@ export class InsightsService {
constructor(private client: ClickHouseClient) {}
private async getTrafficSpikes(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'referrer_name',
'toDate(created_at) as date',
@@ -100,7 +100,7 @@ export class InsightsService {
}
private async getEventSurges(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toDate(created_at) as date',
'COUNT(*) as event_count',
@@ -126,7 +126,7 @@ export class InsightsService {
}
private async getNewVisitorTrends(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toMonth(created_at) as month',
'COUNT(DISTINCT device_id) as new_visitors',
@@ -155,7 +155,7 @@ export class InsightsService {
private async getReferralSourceHighlights(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'referrer_name',
'COUNT(*) as count',
@@ -179,7 +179,7 @@ export class InsightsService {
private async getSessionDurationChanges(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toWeek(created_at) as week',
'avg(duration) as avg_duration',
@@ -205,7 +205,7 @@ export class InsightsService {
}
private async getTopPerformingContent(projectId: string): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'path',
'COUNT(*) as view_count',
@@ -233,7 +233,7 @@ export class InsightsService {
private async getBounceRateImprovements(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toMonth(created_at) as month',
'sum(is_bounce) / COUNT(*) as bounce_rate',
@@ -261,7 +261,7 @@ export class InsightsService {
private async getReturningVisitorTrends(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'toQuarter(created_at) as quarter',
'COUNT(DISTINCT device_id) as returning_visitors',
@@ -290,7 +290,7 @@ export class InsightsService {
private async getGeographicInterestShifts(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'country',
'COUNT(*) as visitor_count',
@@ -318,7 +318,7 @@ export class InsightsService {
private async getEventCompletionChanges(
projectId: string,
): Promise<Insight[]> {
const query = createQuery(this.client)
const query = clix(this.client)
.select([
'event_name',
'toMonth(created_at) as month',

39
packages/db/src/services/organization.service.ts
View File

@@ -14,10 +14,6 @@ export type IServiceMember = Prisma.MemberGetPayload<{
}> & { access: ProjectAccess[] };
export type IServiceProjectAccess = ProjectAccess;
export function transformOrganization<T>(org: T) {
return org;
}
export async function getOrganizations(userId: string | null) {
if (!userId) return [];
@@ -34,10 +30,10 @@ export async function getOrganizations(userId: string | null) {
},
});
return organizations.map(transformOrganization);
return organizations;
}
export function getOrganizationBySlug(slug: string) {
export function getOrganizationById(slug: string) {
return db.organization.findUniqueOrThrow({
where: {
id: slug,
@@ -59,7 +55,7 @@ export async function getOrganizationByProjectId(projectId: string) {
return null;
}
return transformOrganization(project.organization);
return project.organization;
}
export const getOrganizationByProjectIdCached = cacheable(
@@ -258,3 +254,32 @@ export async function getOrganizationSubscriptionChartEndDate(
return endDate;
}
const DEFAULT_TIMEZONE = 'UTC';
export async function getSettingsForOrganization(organizationId: string) {
const organization = await db.organization.findUniqueOrThrow({
where: {
id: organizationId,
},
});
return {
timezone: organization.timezone || DEFAULT_TIMEZONE,
};
}
export async function getSettingsForProject(projectId: string) {
const project = await db.project.findUniqueOrThrow({
where: {
id: projectId,
},
include: {
organization: true,
},
});
return {
timezone: project.organization.timezone || DEFAULT_TIMEZONE,
};
}

133
packages/db/src/services/overview.service.ts
View File

@@ -15,7 +15,9 @@ export const zGetMetricsInput = z.object({
interval: zTimeInterval,
});
export type IGetMetricsInput = z.infer<typeof zGetMetricsInput>;
export type IGetMetricsInput = z.infer<typeof zGetMetricsInput> & {
timezone: string;
};
export const zGetTopPagesInput = z.object({
projectId: z.string(),
@@ -27,7 +29,9 @@ export const zGetTopPagesInput = z.object({
limit: z.number().optional(),
});
export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput>;
export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput> & {
timezone: string;
};
export const zGetTopEntryExitInput = z.object({
projectId: z.string(),
@@ -40,7 +44,9 @@ export const zGetTopEntryExitInput = z.object({
limit: z.number().optional(),
});
export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput>;
export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput> & {
timezone: string;
};
export const zGetTopGenericInput = z.object({
projectId: z.string(),
@@ -75,7 +81,9 @@ export const zGetTopGenericInput = z.object({
limit: z.number().optional(),
});
export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput>;
export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput> & {
timezone: string;
};
export class OverviewService {
private pendingQueries: Map<string, Promise<number | null>> = new Map();
@@ -91,11 +99,13 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
}: {
projectId: string;
startDate: string;
endDate: string;
filters: IChartEventFilter[];
timezone: string;
}) {
const where = this.getRawWhereClause('sessions', filters);
const key = `total_sessions_${projectId}_${startDate}_${endDate}_${JSON.stringify(filters)}`;
@@ -109,15 +119,15 @@ export class OverviewService {
// Create new query promise and store it
const queryPromise = getCache(key, 15, async () => {
try {
const result = await clix(this.client)
const result = await clix(this.client, timezone)
.select<{
total_sessions: number;
}>(['sum(sign) as total_sessions'])
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(where)
.having('sum(sign)', '>', 0)
@@ -138,6 +148,7 @@ export class OverviewService {
startDate,
endDate,
interval,
timezone,
}: IGetMetricsInput): Promise<{
metrics: {
bounce_rate: number;
@@ -160,17 +171,17 @@ export class OverviewService {
const where = this.getRawWhereClause('sessions', filters);
if (this.isPageFilter(filters)) {
// Session aggregation with bounce rates
const sessionAggQuery = clix(this.client)
const sessionAggQuery = clix(this.client, timezone)
.select([
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
`${clix.toStartOf('created_at', interval, timezone)} AS date`,
'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
])
.from(TABLE_NAMES.sessions, true)
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(where)
.groupBy(['date'])
@@ -178,7 +189,7 @@ export class OverviewService {
.orderBy('date', 'ASC');
// Overall unique visitors
const overallUniqueVisitorsQuery = clix(this.client)
const overallUniqueVisitorsQuery = clix(this.client, timezone)
.select([
'uniq(profile_id) AS unique_visitors',
'uniq(session_id) AS total_sessions',
@@ -187,23 +198,23 @@ export class OverviewService {
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters));
return clix(this.client)
return clix(this.client, timezone)
.with('session_agg', sessionAggQuery)
.with(
'overall_bounce_rate',
clix(this.client)
clix(this.client, timezone)
.select(['bounce_rate'])
.from('session_agg')
.where('date', '=', clix.exp("'1970-01-01 00:00:00'")),
)
.with(
'daily_stats',
clix(this.client)
clix(this.client, timezone)
.select(['date', 'bounce_rate'])
.from('session_agg')
.where('date', '!=', clix.exp("'1970-01-01 00:00:00'")),
@@ -221,7 +232,7 @@ export class OverviewService {
overall_total_sessions: number;
overall_bounce_rate: number;
}>([
`${clix.toStartOfInterval('e.created_at', interval, startDate)} AS date`,
`${clix.toInterval('e.created_at', interval)} AS date`,
'ds.bounce_rate as bounce_rate',
'uniq(e.profile_id) AS unique_visitors',
'uniq(e.session_id) AS total_sessions',
@@ -236,20 +247,29 @@ export class OverviewService {
.from(`${TABLE_NAMES.events} AS e`)
.leftJoin(
'daily_stats AS ds',
`${clix.toStartOfInterval('e.created_at', interval, startDate)} = ds.date`,
`${clix.toInterval('e.created_at', interval)} = ds.date`,
)
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters))
.groupBy(['date', 'ds.bounce_rate'])
.orderBy('date', 'ASC')
.fill(
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
clix.toStartOf(
clix.datetime(
startDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
interval,
),
clix.datetime(
endDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
clix.toInterval('1', interval),
)
.transform({
@@ -289,7 +309,7 @@ export class OverviewService {
});
}
const query = clix(this.client)
const query = clix(this.client, timezone)
.select<{
date: string;
bounce_rate: number;
@@ -299,7 +319,7 @@ export class OverviewService {
total_screen_views: number;
views_per_session: number;
}>([
`${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
`${clix.toStartOf('created_at', interval, timezone)} AS date`,
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
'uniqIf(profile_id, sign > 0) AS unique_visitors',
'sum(sign) AS total_sessions',
@@ -310,8 +330,8 @@ export class OverviewService {
])
.from('sessions')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.where('project_id', '=', projectId)
.rawWhere(where)
@@ -320,8 +340,17 @@ export class OverviewService {
.rollup()
.orderBy('date', 'ASC')
.fill(
clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
clix.toStartOf(
clix.datetime(
startDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
interval,
),
clix.datetime(
endDate,
['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
),
clix.toInterval('1', interval),
)
.transform({
@@ -384,8 +413,9 @@ export class OverviewService {
endDate,
cursor = 1,
limit = 10,
timezone,
}: IGetTopPagesInput) {
const pageStatsQuery = clix(this.client)
const pageStatsQuery = clix(this.client, timezone)
.select([
'origin',
'path',
@@ -398,15 +428,15 @@ export class OverviewService {
.where('name', '=', 'screen_view')
.where('path', '!=', '')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy(['origin', 'path'])
.orderBy('count', 'DESC')
.limit(limit)
.offset((cursor - 1) * limit);
const bounceStatsQuery = clix(this.client)
const bounceStatsQuery = clix(this.client, timezone)
.select([
'entry_path',
'entry_origin',
@@ -416,15 +446,15 @@ export class OverviewService {
.where('sign', '=', 1)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy(['entry_path', 'entry_origin']);
pageStatsQuery.rawWhere(this.getRawWhereClause('events', filters));
bounceStatsQuery.rawWhere(this.getRawWhereClause('sessions', filters));
const mainQuery = clix(this.client)
const mainQuery = clix(this.client, timezone)
.with('page_stats', pageStatsQuery)
.with('bounce_stats', bounceStatsQuery)
.select<{
@@ -455,6 +485,7 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
});
return mainQuery.execute();
@@ -468,6 +499,7 @@ export class OverviewService {
mode,
cursor = 1,
limit = 10,
timezone,
}: IGetTopEntryExitInput) {
const where = this.getRawWhereClause('sessions', filters);
@@ -476,11 +508,12 @@ export class OverviewService {
filters,
startDate,
endDate,
timezone,
});
const offset = (cursor - 1) * limit;
const query = clix(this.client)
const query = clix(this.client, timezone)
.select<{
origin: string;
path: string;
@@ -497,8 +530,8 @@ export class OverviewService {
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(where)
.groupBy([`${mode}_origin`, `${mode}_path`])
@@ -510,7 +543,7 @@ export class OverviewService {
let mainQuery = query;
if (this.isPageFilter(filters)) {
mainQuery = clix(this.client)
mainQuery = clix(this.client, timezone)
.with('distinct_sessions', distinctSessionQuery)
.merge(query)
.where(
@@ -525,6 +558,7 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
});
return mainQuery.execute();
@@ -535,19 +569,21 @@ export class OverviewService {
filters,
startDate,
endDate,
timezone,
}: {
projectId: string;
filters: IChartEventFilter[];
startDate: string;
endDate: string;
timezone: string;
}) {
return clix(this.client)
return clix(this.client, timezone)
.select(['DISTINCT session_id'])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters));
}
@@ -560,12 +596,14 @@ export class OverviewService {
column,
cursor = 1,
limit = 10,
timezone,
}: IGetTopGenericInput) {
const distinctSessionQuery = this.getDistinctSessions({
projectId,
filters,
startDate,
endDate,
timezone,
});
const prefixColumn = (() => {
@@ -584,7 +622,7 @@ export class OverviewService {
const offset = (cursor - 1) * limit;
const query = clix(this.client)
const query = clix(this.client, timezone)
.select<{
prefix?: string;
name: string;
@@ -601,8 +639,8 @@ export class OverviewService {
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
.where('created_at', 'BETWEEN', [
clix.datetime(startDate),
clix.datetime(endDate),
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.groupBy([prefixColumn, column].filter(Boolean))
.having('sum(sign)', '>', 0)
@@ -613,7 +651,7 @@ export class OverviewService {
let mainQuery = query;
if (this.isPageFilter(filters)) {
mainQuery = clix(this.client)
mainQuery = clix(this.client, timezone)
.with('distinct_sessions', distinctSessionQuery)
.merge(query)
.where(
@@ -632,6 +670,7 @@ export class OverviewService {
startDate,
endDate,
filters,
timezone,
}),
]);