zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add better support for timezones

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-05-02 21:16:31 +02:00
parent cff7822f12
commit 33e92a0da0
9 changed files with 152 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/index.ts
View File

@@ -56,8 +56,8 @@ const startServer = async () => {
onError(error: unknown) {
if (error instanceof Error) {
logger.error(error, error.message);
} else {
logger.error(error, 'Unknown error trpc error');
} else if (error && typeof error === 'object' && 'error' in error) {
logger.error(error.error, 'Unknown error trpc error');
}
},
} satisfies FastifyTRPCPluginOptions<AppRouter>['trpcOptions'],

11
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx
View File

@@ -2,6 +2,7 @@
import { useOverviewOptions } from '@/components/overview/useOverviewOptions';
import { TimeWindowPicker } from '@/components/time-window-picker';
import { endOfDay, formatISO, startOfDay } from 'date-fns';
export function OverviewReportRange() {
const { range, setRange, setStartDate, setEndDate, endDate, startDate } =
@@ -11,8 +12,14 @@ export function OverviewReportRange() {
<TimeWindowPicker
onChange={setRange}
value={range}
onStartDateChange={(date) => setStartDate(date)}
onEndDateChange={(date) => setEndDate(date)}
onStartDateChange={(date) => {
const d = formatISO(startOfDay(new Date(date)));
setStartDate(d);
}}
onEndDateChange={(date) => {
const d = formatISO(endOfDay(new Date(date)));
setEndDate(d);
}}
endDate={endDate}
startDate={startDate}
/>

3
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx
View File

@@ -1,3 +0,0 @@
import FullPageLoadingState from '@/components/full-page-loading-state';
export default FullPageLoadingState;

16
apps/dashboard/src/components/report/reportSlice.ts
View File

@@ -1,6 +1,12 @@
import { createSlice } from '@reduxjs/toolkit';
import type { PayloadAction } from '@reduxjs/toolkit';
import { isSameDay, isSameMonth } from 'date-fns';
import {
endOfDay,
formatISO,
isSameDay,
isSameMonth,
startOfDay,
} from 'date-fns';
import {
alphabetIds,
@@ -188,8 +194,8 @@ export const reportSlice = createSlice({
}>
) => {
state.dirty = true;
state.startDate = action.payload.startDate;
state.endDate = action.payload.endDate;
state.startDate = formatISO(startOfDay(action.payload.startDate));
state.endDate = formatISO(endOfDay(action.payload.endDate));
if (isSameDay(state.startDate, state.endDate)) {
state.interval = 'hour';
@@ -203,7 +209,7 @@ export const reportSlice = createSlice({
// Date range
changeStartDate: (state, action: PayloadAction<string>) => {
state.dirty = true;
state.startDate = action.payload;
state.startDate = formatISO(startOfDay(action.payload));
const interval = getDefaultIntervalByDates(
state.startDate,
@@ -217,7 +223,7 @@ export const reportSlice = createSlice({
// Date range
changeEndDate: (state, action: PayloadAction<string>) => {
state.dirty = true;
state.endDate = action.payload;
state.endDate = formatISO(endOfDay(action.payload));
const interval = getDefaultIntervalByDates(
state.startDate,

38
packages/common/src/date.ts
View File

@@ -5,3 +5,41 @@ export function getTime(date: string | number | Date) {
export function toISOString(date: string | number | Date) {
return new Date(date).toISOString();
}
export function getTimezoneFromDateString(_date: string) {
const mapper: Record<string, string> = {
'+00:00': 'UTC',
'+01:00': 'Europe/Paris',
'+02:00': 'Europe/Stockholm',
'+03:00': 'Europe/Moscow',
'+04:00': 'Asia/Dubai',
'+05:00': 'Asia/Karachi',
'+06:00': 'Asia/Dhaka',
'+07:00': 'Asia/Bangkok',
'+08:00': 'Asia/Shanghai',
'+09:00': 'Asia/Tokyo',
'+10:00': 'Australia/Sydney',
'+11:00': 'Pacific/Noumea',
'+12:00': 'Pacific/Fiji',
'-02:00': 'America/Noronha',
'-03:00': 'America/Sao_Paulo',
'-04:00': 'America/Santiago',
'-05:00': 'America/Bogota',
'-06:00': 'America/Mexico_City',
'-07:00': 'America/Phoenix',
'-08:00': 'America/Los_Angeles',
'-09:00': 'America/Anchorage',
'-10:00': 'Pacific/Honolulu',
'-11:00': 'Pacific/Midway',
'-12:00': 'Pacific/Tarawa',
};
const defaultTimezone = 'UTC';
const match = _date.match(/([+-][0-9]{2}):([0-9]{2})$/)?.[0];
if (match) {
return mapper[match] ?? defaultTimezone;
}
return defaultTimezone;
}

2
packages/constants/index.ts
View File

@@ -15,7 +15,7 @@ export const timeWindows = {
},
today: {
key: 'today',
label: 'Today',
label: '24 hours',
shortcut: 'D',
},
'7d': {

8
packages/db/src/clickhouse-client.ts
View File

@@ -15,6 +15,7 @@ export const ch = createClient({
export async function chQueryWithMeta<T extends Record<string, any>>(
query: string
): Promise<ResponseJSON<T>> {
console.log('Query:', query);
const start = Date.now();
const res = await ch.query({
query,
@@ -39,7 +40,6 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
console.log(`Clickhouse query took ${response.statistics?.elapsed}ms`);
console.log(`chQuery took ${Date.now() - start}ms`);
console.log('Query:', query);
return response;
}
@@ -52,7 +52,11 @@ export async function chQuery<T extends Record<string, any>>(
export function formatClickhouseDate(_date: Date | string) {
const date = typeof _date === 'string' ? new Date(_date) : _date;
return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
return date
.toISOString()
.replace('T', ' ')
.replace(/Z+$/, '')
.replace(/\.[0-9]+$/, '');
}
export function convertClickhouseDateToJs(date: string) {

23
packages/db/src/services/chart.service.ts
View File

@@ -1,5 +1,6 @@
import { escape } from 'sqlstring';
import { getTimezoneFromDateString } from '@openpanel/common';
import type {
IChartEventFilter,
IGetChartDataInput,
@@ -21,32 +22,40 @@ export function getChartSql({
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.projectId = `project_id = ${escape(projectId)}`;
let labelValue = escape('*');
if (event.name !== '*') {
sb.select.label = `${escape(event.name)} as label`;
sb.where.eventName = `name = ${escape(event.name)}`;
labelValue = `${escape(event.name)}`;
sb.select.label = `${labelValue} as label`;
sb.where.eventName = `name = ${labelValue}`;
} else {
sb.select.label = `${labelValue} as label`;
}
sb.select.count = `count(*) as count`;
switch (interval) {
case 'minute': {
sb.select.date = `toStartOfMinute(created_at) as date`;
sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'hour': {
sb.select.date = `toStartOfHour(created_at) as date`;
sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'day': {
sb.select.date = `toStartOfDay(created_at) as date`;
sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'month': {
sb.select.date = `toStartOfMonth(created_at) as date`;
sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
}
sb.groupBy.date = 'date';
sb.orderBy.date = 'date ASC';
if (startDate) {
sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;

236
packages/trpc/src/routers/chart.helpers.ts
View File

@@ -1,10 +1,14 @@
import {
differenceInMilliseconds,
endOfDay,
endOfMonth,
endOfYear,
formatISO,
startOfDay,
startOfMonth,
startOfYear,
subDays,
subMilliseconds,
subMinutes,
subMonths,
subYears,
@@ -43,129 +47,6 @@ function getEventLegend(event: IChartEvent) {
return event.displayName ?? `${event.name} (${event.id})`;
}
function fillEmptySpotsInTimeline(
items: ResultItem[],
interval: IInterval,
startDate: string,
endDate: string
) {
const result = [];
const clonedStartDate = new Date(startDate);
const clonedEndDate = new Date(endDate);
const today = new Date();
if (interval === 'minute') {
clonedStartDate.setUTCSeconds(0, 0);
clonedEndDate.setUTCMinutes(clonedEndDate.getUTCMinutes() + 1, 0, 0);
} else if (interval === 'hour') {
clonedStartDate.setUTCMinutes(0, 0, 0);
clonedEndDate.setUTCMinutes(0, 0, 0);
} else {
clonedStartDate.setUTCHours(0, 0, 0, 0);
clonedEndDate.setUTCHours(0, 0, 0, 0);
}
if (interval === 'month') {
clonedStartDate.setUTCDate(1);
clonedEndDate.setUTCDate(1);
}
// Force if interval is month and the start date is the same month as today
const shouldForce = () =>
interval === 'month' &&
clonedStartDate.getUTCFullYear() === today.getUTCFullYear() &&
clonedStartDate.getUTCMonth() === today.getUTCMonth();
let prev = undefined;
while (
shouldForce() ||
clonedStartDate.getTime() <= clonedEndDate.getTime()
) {
if (prev === clonedStartDate.getTime()) {
break;
}
prev = clonedStartDate.getTime();
const getYear = (date: Date) => date.getUTCFullYear();
const getMonth = (date: Date) => date.getUTCMonth();
const getDay = (date: Date) => date.getUTCDate();
const getHour = (date: Date) => date.getUTCHours();
const getMinute = (date: Date) => date.getUTCMinutes();
const item = items.find((item) => {
const date = convertClickhouseDateToJs(item.date);
if (interval === 'month') {
return (
getYear(date) === getYear(clonedStartDate) &&
getMonth(date) === getMonth(clonedStartDate)
);
}
if (interval === 'day') {
return (
getYear(date) === getYear(clonedStartDate) &&
getMonth(date) === getMonth(clonedStartDate) &&
getDay(date) === getDay(clonedStartDate)
);
}
if (interval === 'hour') {
return (
getYear(date) === getYear(clonedStartDate) &&
getMonth(date) === getMonth(clonedStartDate) &&
getDay(date) === getDay(clonedStartDate) &&
getHour(date) === getHour(clonedStartDate)
);
}
if (interval === 'minute') {
return (
getYear(date) === getYear(clonedStartDate) &&
getMonth(date) === getMonth(clonedStartDate) &&
getDay(date) === getDay(clonedStartDate) &&
getHour(date) === getHour(clonedStartDate) &&
getMinute(date) === getMinute(clonedStartDate)
);
}
return false;
});
if (item) {
result.push({
...item,
date: clonedStartDate.toISOString(),
});
} else {
result.push({
date: clonedStartDate.toISOString(),
count: 0,
label: null,
});
}
switch (interval) {
case 'day': {
clonedStartDate.setUTCDate(clonedStartDate.getUTCDate() + 1);
break;
}
case 'hour': {
clonedStartDate.setUTCHours(clonedStartDate.getUTCHours() + 1);
break;
}
case 'minute': {
clonedStartDate.setUTCMinutes(clonedStartDate.getUTCMinutes() + 1);
break;
}
case 'month': {
clonedStartDate.setUTCMonth(clonedStartDate.getUTCMonth() + 1);
break;
}
}
}
return sort(function (a, b) {
return new Date(a.date).getTime() - new Date(b.date).getTime();
}, result);
}
export function withFormula(
{ formula, events }: IChartInput,
series: GetChartDataResult
@@ -235,6 +116,26 @@ export function withFormula(
];
}
const toDynamicISODateWithTZ = (
date: string,
blueprint: string,
interval: IInterval
) => {
// If we have a space in the date we know it's a date with time
if (date.includes(' ')) {
// If interval is minutes we need to convert the timezone to what timezone is used (either on client or the server)
// - We use timezone from server if its a predefined range (yearToDate, lastYear, etc.)
// - We use timezone from client if its a custom range
if (interval === 'minute' || interval === 'hour') {
return date.replace(' ', 'T') + blueprint.slice(-6);
}
// Otherwise we just return without the timezone
// It will be converted to the correct timezone on the client
return date.replace(' ', 'T');
}
return `${date}T00:00:00`;
};
export async function getChartData(payload: IGetChartDataInput) {
let result = await chQuery<ResultItem>(getChartSql(payload));
@@ -250,9 +151,15 @@ export async function getChartData(payload: IGetChartDataInput) {
// group by sql label
const series = result.reduce(
(acc, item) => {
// If we fill empty spots in the timeline (clickhouse) we wont get a label back
// take the event name as label
if (!item.label && item.count === 0) {
item.label = payload.event.name;
}
const label = item.label?.trim() || NOT_SET_VALUE;
// item.label can be null when using breakdowns on a property
// that doesn't exist on all events
const label = item.label?.trim() || NOT_SET_VALUE;
if (label) {
if (acc[label]) {
acc[label]?.push(item);
@@ -281,22 +188,25 @@ export async function getChartData(payload: IGetChartDataInput) {
payload.chartType === 'metric' ||
payload.chartType === 'pie' ||
payload.chartType === 'bar'
? fillEmptySpotsInTimeline(
series[key] ?? [],
payload.interval,
payload.startDate,
payload.endDate
).map((item) => {
? (series[key] ?? []).map((item) => {
return {
label: serieName,
count: item.count ? round(item.count) : null,
date: new Date(item.date).toISOString(),
date: toDynamicISODateWithTZ(
item.date,
payload.startDate,
payload.interval
),
};
})
: (series[key] ?? []).map((item) => ({
label: item.label,
count: item.count ? round(item.count) : null,
date: new Date(item.date).toISOString(),
date: toDynamicISODateWithTZ(
item.date,
payload.startDate,
payload.interval
),
}));
return {
@@ -310,8 +220,8 @@ export async function getChartData(payload: IGetChartDataInput) {
export function getDatesFromRange(range: IChartRange) {
if (range === '30min' || range === 'lastHour') {
const minutes = range === '30min' ? 30 : 60;
const startDate = subMinutes(new Date(), minutes).toUTCString();
const endDate = new Date().toUTCString();
const startDate = formatISO(subMinutes(new Date(), minutes));
const endDate = formatISO(new Date());
return {
startDate,
@@ -320,18 +230,22 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === 'today') {
const startDate = startOfDay(new Date());
const endDate = endOfDay(new Date());
// This is last 24 hours instead
// Makes it easier to handle timezones
// const startDate = startOfDay(new Date());
// const endDate = endOfDay(new Date());
const startDate = subDays(new Date(), 1);
const endDate = new Date();
return {
startDate: startDate.toUTCString(),
endDate: endDate.toUTCString(),
startDate: formatISO(startDate),
endDate: formatISO(endDate),
};
}
if (range === '7d') {
const startDate = subDays(new Date(), 7).toUTCString();
const endDate = new Date().toUTCString();
const startDate = formatISO(subDays(new Date(), 7));
const endDate = formatISO(new Date());
return {
startDate,
@@ -340,8 +254,8 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === '30d') {
const startDate = subDays(new Date(), 30).toUTCString();
const endDate = new Date().toUTCString();
const startDate = formatISO(subDays(new Date(), 30));
const endDate = formatISO(new Date());
return {
startDate,
@@ -350,8 +264,8 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === 'monthToDate') {
const startDate = startOfMonth(new Date()).toUTCString();
const endDate = new Date().toUTCString();
const startDate = formatISO(startOfMonth(new Date()));
const endDate = formatISO(new Date());
return {
startDate,
@@ -361,8 +275,8 @@ export function getDatesFromRange(range: IChartRange) {
if (range === 'lastMonth') {
const month = subMonths(new Date(), 1);
const startDate = startOfMonth(month).toUTCString();
const endDate = endOfDay(month).toUTCString();
const startDate = formatISO(startOfMonth(month));
const endDate = formatISO(endOfMonth(month));
return {
startDate,
@@ -371,8 +285,8 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === 'yearToDate') {
const startDate = startOfYear(new Date()).toUTCString();
const endDate = new Date().toUTCString();
const startDate = formatISO(startOfYear(new Date()));
const endDate = formatISO(new Date());
return {
startDate,
@@ -382,8 +296,8 @@ export function getDatesFromRange(range: IChartRange) {
if (range === 'lastYear') {
const year = subYears(new Date(), 1);
const startDate = startOfYear(year).toUTCString();
const endDate = endOfYear(year).toUTCString();
const startDate = formatISO(startOfYear(year));
const endDate = formatISO(endOfYear(year));
return {
startDate,
@@ -391,20 +305,9 @@ export function getDatesFromRange(range: IChartRange) {
};
}
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
console.log('-------------------------------');
return {
startDate: subDays(new Date(), 30).toISOString(),
endDate: new Date().toISOString(),
startDate: formatISO(subDays(new Date(), 30)),
endDate: formatISO(new Date()),
};
}
@@ -421,16 +324,15 @@ export function getChartStartEndDate({
export function getChartPrevStartEndDate({
startDate,
endDate,
range,
}: {
startDate: string;
endDate: string;
range: IChartRange;
}) {
const diff = new Date(endDate).getTime() - new Date(startDate).getTime();
const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate));
return {
startDate: new Date(new Date(startDate).getTime() - diff).toISOString(),
endDate: new Date(new Date(endDate).getTime() - diff).toISOString(),
startDate: formatISO(subMilliseconds(new Date(startDate), diff - 1)),
endDate: formatISO(subMilliseconds(new Date(endDate), diff - 1)),
};
}