diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 1f98d79e..2d698e35 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -56,8 +56,8 @@ const startServer = async () => { onError(error: unknown) { if (error instanceof Error) { logger.error(error, error.message); - } else { - logger.error(error, 'Unknown error trpc error'); + } else if (error && typeof error === 'object' && 'error' in error) { + logger.error(error.error, 'Unknown error trpc error'); } }, } satisfies FastifyTRPCPluginOptions['trpcOptions'], diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx index dbc335d2..6722fda9 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx @@ -2,6 +2,7 @@ import { useOverviewOptions } from '@/components/overview/useOverviewOptions'; import { TimeWindowPicker } from '@/components/time-window-picker'; +import { endOfDay, formatISO, startOfDay } from 'date-fns'; export function OverviewReportRange() { const { range, setRange, setStartDate, setEndDate, endDate, startDate } = @@ -11,8 +12,14 @@ export function OverviewReportRange() { setStartDate(date)} - onEndDateChange={(date) => setEndDate(date)} + onStartDateChange={(date) => { + const d = formatISO(startOfDay(new Date(date))); + setStartDate(d); + }} + onEndDateChange={(date) => { + const d = formatISO(endOfDay(new Date(date))); + setEndDate(d); + }} endDate={endDate} startDate={startDate} /> diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx deleted file mode 100644 index 9cf2cf50..00000000 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx +++ /dev/null @@ -1,3 +0,0 @@ -import FullPageLoadingState from '@/components/full-page-loading-state'; - -export default FullPageLoadingState; diff --git a/apps/dashboard/src/components/report/reportSlice.ts b/apps/dashboard/src/components/report/reportSlice.ts index 0616a331..b6321128 100644 --- a/apps/dashboard/src/components/report/reportSlice.ts +++ b/apps/dashboard/src/components/report/reportSlice.ts @@ -1,6 +1,12 @@ import { createSlice } from '@reduxjs/toolkit'; import type { PayloadAction } from '@reduxjs/toolkit'; -import { isSameDay, isSameMonth } from 'date-fns'; +import { + endOfDay, + formatISO, + isSameDay, + isSameMonth, + startOfDay, +} from 'date-fns'; import { alphabetIds, @@ -188,8 +194,8 @@ export const reportSlice = createSlice({ }> ) => { state.dirty = true; - state.startDate = action.payload.startDate; - state.endDate = action.payload.endDate; + state.startDate = formatISO(startOfDay(action.payload.startDate)); + state.endDate = formatISO(endOfDay(action.payload.endDate)); if (isSameDay(state.startDate, state.endDate)) { state.interval = 'hour'; @@ -203,7 +209,7 @@ export const reportSlice = createSlice({ // Date range changeStartDate: (state, action: PayloadAction) => { state.dirty = true; - state.startDate = action.payload; + state.startDate = formatISO(startOfDay(action.payload)); const interval = getDefaultIntervalByDates( state.startDate, @@ -217,7 +223,7 @@ export const reportSlice = createSlice({ // Date range changeEndDate: (state, action: PayloadAction) => { state.dirty = true; - state.endDate = action.payload; + state.endDate = formatISO(endOfDay(action.payload)); const interval = getDefaultIntervalByDates( state.startDate, diff --git a/packages/common/src/date.ts b/packages/common/src/date.ts index b2bbdf8d..2f520cc2 100644 --- a/packages/common/src/date.ts +++ b/packages/common/src/date.ts @@ -5,3 +5,41 @@ export function getTime(date: string | number | Date) { export function toISOString(date: string | number | Date) { return new Date(date).toISOString(); } + +export function getTimezoneFromDateString(_date: string) { + const mapper: Record = { + '+00:00': 'UTC', + '+01:00': 'Europe/Paris', + '+02:00': 'Europe/Stockholm', + '+03:00': 'Europe/Moscow', + '+04:00': 'Asia/Dubai', + '+05:00': 'Asia/Karachi', + '+06:00': 'Asia/Dhaka', + '+07:00': 'Asia/Bangkok', + '+08:00': 'Asia/Shanghai', + '+09:00': 'Asia/Tokyo', + '+10:00': 'Australia/Sydney', + '+11:00': 'Pacific/Noumea', + '+12:00': 'Pacific/Fiji', + '-02:00': 'America/Noronha', + '-03:00': 'America/Sao_Paulo', + '-04:00': 'America/Santiago', + '-05:00': 'America/Bogota', + '-06:00': 'America/Mexico_City', + '-07:00': 'America/Phoenix', + '-08:00': 'America/Los_Angeles', + '-09:00': 'America/Anchorage', + '-10:00': 'Pacific/Honolulu', + '-11:00': 'Pacific/Midway', + '-12:00': 'Pacific/Tarawa', + }; + + const defaultTimezone = 'UTC'; + + const match = _date.match(/([+-][0-9]{2}):([0-9]{2})$/)?.[0]; + if (match) { + return mapper[match] ?? defaultTimezone; + } + + return defaultTimezone; +} diff --git a/packages/constants/index.ts b/packages/constants/index.ts index c2bf653f..acd51f94 100644 --- a/packages/constants/index.ts +++ b/packages/constants/index.ts @@ -15,7 +15,7 @@ export const timeWindows = { }, today: { key: 'today', - label: 'Today', + label: '24 hours', shortcut: 'D', }, '7d': { diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.ts index 2d3064cd..3966c755 100644 --- a/packages/db/src/clickhouse-client.ts +++ b/packages/db/src/clickhouse-client.ts @@ -15,6 +15,7 @@ export const ch = createClient({ export async function chQueryWithMeta>( query: string ): Promise> { + console.log('Query:', query); const start = Date.now(); const res = await ch.query({ query, @@ -39,7 +40,6 @@ export async function chQueryWithMeta>( console.log(`Clickhouse query took ${response.statistics?.elapsed}ms`); console.log(`chQuery took ${Date.now() - start}ms`); - console.log('Query:', query); return response; } @@ -52,7 +52,11 @@ export async function chQuery>( export function formatClickhouseDate(_date: Date | string) { const date = typeof _date === 'string' ? new Date(_date) : _date; - return date.toISOString().replace('T', ' ').replace(/Z+$/, ''); + return date + .toISOString() + .replace('T', ' ') + .replace(/Z+$/, '') + .replace(/\.[0-9]+$/, ''); } export function convertClickhouseDateToJs(date: string) { diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts index a49dd787..be56c933 100644 --- a/packages/db/src/services/chart.service.ts +++ b/packages/db/src/services/chart.service.ts @@ -1,5 +1,6 @@ import { escape } from 'sqlstring'; +import { getTimezoneFromDateString } from '@openpanel/common'; import type { IChartEventFilter, IGetChartDataInput, @@ -21,32 +22,40 @@ export function getChartSql({ sb.where = getEventFiltersWhereClause(event.filters); sb.where.projectId = `project_id = ${escape(projectId)}`; + + let labelValue = escape('*'); if (event.name !== '*') { - sb.select.label = `${escape(event.name)} as label`; - sb.where.eventName = `name = ${escape(event.name)}`; + labelValue = `${escape(event.name)}`; + sb.select.label = `${labelValue} as label`; + sb.where.eventName = `name = ${labelValue}`; + } else { + sb.select.label = `${labelValue} as label`; } sb.select.count = `count(*) as count`; switch (interval) { case 'minute': { - sb.select.date = `toStartOfMinute(created_at) as date`; + sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; + sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`; break; } case 'hour': { - sb.select.date = `toStartOfHour(created_at) as date`; + sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; + sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`; break; } case 'day': { - sb.select.date = `toStartOfDay(created_at) as date`; + sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; + sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`; break; } case 'month': { - sb.select.date = `toStartOfMonth(created_at) as date`; + sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`; + sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`; break; } } sb.groupBy.date = 'date'; - sb.orderBy.date = 'date ASC'; if (startDate) { sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`; diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index f2f98c61..44a4d9f3 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -1,10 +1,14 @@ import { + differenceInMilliseconds, endOfDay, + endOfMonth, endOfYear, + formatISO, startOfDay, startOfMonth, startOfYear, subDays, + subMilliseconds, subMinutes, subMonths, subYears, @@ -43,129 +47,6 @@ function getEventLegend(event: IChartEvent) { return event.displayName ?? `${event.name} (${event.id})`; } -function fillEmptySpotsInTimeline( - items: ResultItem[], - interval: IInterval, - startDate: string, - endDate: string -) { - const result = []; - const clonedStartDate = new Date(startDate); - const clonedEndDate = new Date(endDate); - const today = new Date(); - - if (interval === 'minute') { - clonedStartDate.setUTCSeconds(0, 0); - clonedEndDate.setUTCMinutes(clonedEndDate.getUTCMinutes() + 1, 0, 0); - } else if (interval === 'hour') { - clonedStartDate.setUTCMinutes(0, 0, 0); - clonedEndDate.setUTCMinutes(0, 0, 0); - } else { - clonedStartDate.setUTCHours(0, 0, 0, 0); - clonedEndDate.setUTCHours(0, 0, 0, 0); - } - - if (interval === 'month') { - clonedStartDate.setUTCDate(1); - clonedEndDate.setUTCDate(1); - } - - // Force if interval is month and the start date is the same month as today - const shouldForce = () => - interval === 'month' && - clonedStartDate.getUTCFullYear() === today.getUTCFullYear() && - clonedStartDate.getUTCMonth() === today.getUTCMonth(); - let prev = undefined; - while ( - shouldForce() || - clonedStartDate.getTime() <= clonedEndDate.getTime() - ) { - if (prev === clonedStartDate.getTime()) { - break; - } - prev = clonedStartDate.getTime(); - - const getYear = (date: Date) => date.getUTCFullYear(); - const getMonth = (date: Date) => date.getUTCMonth(); - const getDay = (date: Date) => date.getUTCDate(); - const getHour = (date: Date) => date.getUTCHours(); - const getMinute = (date: Date) => date.getUTCMinutes(); - - const item = items.find((item) => { - const date = convertClickhouseDateToJs(item.date); - - if (interval === 'month') { - return ( - getYear(date) === getYear(clonedStartDate) && - getMonth(date) === getMonth(clonedStartDate) - ); - } - if (interval === 'day') { - return ( - getYear(date) === getYear(clonedStartDate) && - getMonth(date) === getMonth(clonedStartDate) && - getDay(date) === getDay(clonedStartDate) - ); - } - if (interval === 'hour') { - return ( - getYear(date) === getYear(clonedStartDate) && - getMonth(date) === getMonth(clonedStartDate) && - getDay(date) === getDay(clonedStartDate) && - getHour(date) === getHour(clonedStartDate) - ); - } - if (interval === 'minute') { - return ( - getYear(date) === getYear(clonedStartDate) && - getMonth(date) === getMonth(clonedStartDate) && - getDay(date) === getDay(clonedStartDate) && - getHour(date) === getHour(clonedStartDate) && - getMinute(date) === getMinute(clonedStartDate) - ); - } - - return false; - }); - - if (item) { - result.push({ - ...item, - date: clonedStartDate.toISOString(), - }); - } else { - result.push({ - date: clonedStartDate.toISOString(), - count: 0, - label: null, - }); - } - - switch (interval) { - case 'day': { - clonedStartDate.setUTCDate(clonedStartDate.getUTCDate() + 1); - break; - } - case 'hour': { - clonedStartDate.setUTCHours(clonedStartDate.getUTCHours() + 1); - break; - } - case 'minute': { - clonedStartDate.setUTCMinutes(clonedStartDate.getUTCMinutes() + 1); - break; - } - case 'month': { - clonedStartDate.setUTCMonth(clonedStartDate.getUTCMonth() + 1); - break; - } - } - } - - return sort(function (a, b) { - return new Date(a.date).getTime() - new Date(b.date).getTime(); - }, result); -} - export function withFormula( { formula, events }: IChartInput, series: GetChartDataResult @@ -235,6 +116,26 @@ export function withFormula( ]; } +const toDynamicISODateWithTZ = ( + date: string, + blueprint: string, + interval: IInterval +) => { + // If we have a space in the date we know it's a date with time + if (date.includes(' ')) { + // If interval is minutes we need to convert the timezone to what timezone is used (either on client or the server) + // - We use timezone from server if its a predefined range (yearToDate, lastYear, etc.) + // - We use timezone from client if its a custom range + if (interval === 'minute' || interval === 'hour') { + return date.replace(' ', 'T') + blueprint.slice(-6); + } + // Otherwise we just return without the timezone + // It will be converted to the correct timezone on the client + return date.replace(' ', 'T'); + } + return `${date}T00:00:00`; +}; + export async function getChartData(payload: IGetChartDataInput) { let result = await chQuery(getChartSql(payload)); @@ -250,9 +151,15 @@ export async function getChartData(payload: IGetChartDataInput) { // group by sql label const series = result.reduce( (acc, item) => { + // If we fill empty spots in the timeline (clickhouse) we wont get a label back + // take the event name as label + if (!item.label && item.count === 0) { + item.label = payload.event.name; + } + + const label = item.label?.trim() || NOT_SET_VALUE; // item.label can be null when using breakdowns on a property // that doesn't exist on all events - const label = item.label?.trim() || NOT_SET_VALUE; if (label) { if (acc[label]) { acc[label]?.push(item); @@ -281,22 +188,25 @@ export async function getChartData(payload: IGetChartDataInput) { payload.chartType === 'metric' || payload.chartType === 'pie' || payload.chartType === 'bar' - ? fillEmptySpotsInTimeline( - series[key] ?? [], - payload.interval, - payload.startDate, - payload.endDate - ).map((item) => { + ? (series[key] ?? []).map((item) => { return { label: serieName, count: item.count ? round(item.count) : null, - date: new Date(item.date).toISOString(), + date: toDynamicISODateWithTZ( + item.date, + payload.startDate, + payload.interval + ), }; }) : (series[key] ?? []).map((item) => ({ label: item.label, count: item.count ? round(item.count) : null, - date: new Date(item.date).toISOString(), + date: toDynamicISODateWithTZ( + item.date, + payload.startDate, + payload.interval + ), })); return { @@ -310,8 +220,8 @@ export async function getChartData(payload: IGetChartDataInput) { export function getDatesFromRange(range: IChartRange) { if (range === '30min' || range === 'lastHour') { const minutes = range === '30min' ? 30 : 60; - const startDate = subMinutes(new Date(), minutes).toUTCString(); - const endDate = new Date().toUTCString(); + const startDate = formatISO(subMinutes(new Date(), minutes)); + const endDate = formatISO(new Date()); return { startDate, @@ -320,18 +230,22 @@ export function getDatesFromRange(range: IChartRange) { } if (range === 'today') { - const startDate = startOfDay(new Date()); - const endDate = endOfDay(new Date()); + // This is last 24 hours instead + // Makes it easier to handle timezones + // const startDate = startOfDay(new Date()); + // const endDate = endOfDay(new Date()); + const startDate = subDays(new Date(), 1); + const endDate = new Date(); return { - startDate: startDate.toUTCString(), - endDate: endDate.toUTCString(), + startDate: formatISO(startDate), + endDate: formatISO(endDate), }; } if (range === '7d') { - const startDate = subDays(new Date(), 7).toUTCString(); - const endDate = new Date().toUTCString(); + const startDate = formatISO(subDays(new Date(), 7)); + const endDate = formatISO(new Date()); return { startDate, @@ -340,8 +254,8 @@ export function getDatesFromRange(range: IChartRange) { } if (range === '30d') { - const startDate = subDays(new Date(), 30).toUTCString(); - const endDate = new Date().toUTCString(); + const startDate = formatISO(subDays(new Date(), 30)); + const endDate = formatISO(new Date()); return { startDate, @@ -350,8 +264,8 @@ export function getDatesFromRange(range: IChartRange) { } if (range === 'monthToDate') { - const startDate = startOfMonth(new Date()).toUTCString(); - const endDate = new Date().toUTCString(); + const startDate = formatISO(startOfMonth(new Date())); + const endDate = formatISO(new Date()); return { startDate, @@ -361,8 +275,8 @@ export function getDatesFromRange(range: IChartRange) { if (range === 'lastMonth') { const month = subMonths(new Date(), 1); - const startDate = startOfMonth(month).toUTCString(); - const endDate = endOfDay(month).toUTCString(); + const startDate = formatISO(startOfMonth(month)); + const endDate = formatISO(endOfMonth(month)); return { startDate, @@ -371,8 +285,8 @@ export function getDatesFromRange(range: IChartRange) { } if (range === 'yearToDate') { - const startDate = startOfYear(new Date()).toUTCString(); - const endDate = new Date().toUTCString(); + const startDate = formatISO(startOfYear(new Date())); + const endDate = formatISO(new Date()); return { startDate, @@ -382,8 +296,8 @@ export function getDatesFromRange(range: IChartRange) { if (range === 'lastYear') { const year = subYears(new Date(), 1); - const startDate = startOfYear(year).toUTCString(); - const endDate = endOfYear(year).toUTCString(); + const startDate = formatISO(startOfYear(year)); + const endDate = formatISO(endOfYear(year)); return { startDate, @@ -391,20 +305,9 @@ export function getDatesFromRange(range: IChartRange) { }; } - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); - console.log('-------------------------------'); return { - startDate: subDays(new Date(), 30).toISOString(), - endDate: new Date().toISOString(), + startDate: formatISO(subDays(new Date(), 30)), + endDate: formatISO(new Date()), }; } @@ -421,16 +324,15 @@ export function getChartStartEndDate({ export function getChartPrevStartEndDate({ startDate, endDate, - range, }: { startDate: string; endDate: string; range: IChartRange; }) { - const diff = new Date(endDate).getTime() - new Date(startDate).getTime(); + const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate)); return { - startDate: new Date(new Date(startDate).getTime() - diff).toISOString(), - endDate: new Date(new Date(endDate).getTime() - diff).toISOString(), + startDate: formatISO(subMilliseconds(new Date(startDate), diff - 1)), + endDate: formatISO(subMilliseconds(new Date(endDate), diff - 1)), }; }