add better support for timezones

apps/api/src/index.ts

@@ -56,8 +56,8 @@ const startServer = async () => {
         onError(error: unknown) {
           if (error instanceof Error) {
             logger.error(error, error.message);
-          } else {
+          } else if (error && typeof error === 'object' && 'error' in error) {
-            logger.error(error, 'Unknown error trpc error');
+            logger.error(error.error, 'Unknown error trpc error');
           }
         },
       } satisfies FastifyTRPCPluginOptions<AppRouter>['trpcOptions'],

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx

@@ -2,6 +2,7 @@
 
 import { useOverviewOptions } from '@/components/overview/useOverviewOptions';
 import { TimeWindowPicker } from '@/components/time-window-picker';
+import { endOfDay, formatISO, startOfDay } from 'date-fns';
 
 export function OverviewReportRange() {
   const { range, setRange, setStartDate, setEndDate, endDate, startDate } =
@@ -11,8 +12,14 @@ export function OverviewReportRange() {
     <TimeWindowPicker
       onChange={setRange}
       value={range}
-      onStartDateChange={(date) => setStartDate(date)}
+      onStartDateChange={(date) => {
-      onEndDateChange={(date) => setEndDate(date)}
+        const d = formatISO(startOfDay(new Date(date)));
+        setStartDate(d);
+      }}
+      onEndDateChange={(date) => {
+        const d = formatISO(endOfDay(new Date(date)));
+        setEndDate(d);
+      }}
       endDate={endDate}
       startDate={startDate}
     />

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx

@@ -1,3 +0,0 @@
-import FullPageLoadingState from '@/components/full-page-loading-state';
-
-export default FullPageLoadingState;

apps/dashboard/src/components/report/reportSlice.ts

@@ -1,6 +1,12 @@
 import { createSlice } from '@reduxjs/toolkit';
 import type { PayloadAction } from '@reduxjs/toolkit';
-import { isSameDay, isSameMonth } from 'date-fns';
+import {
+  endOfDay,
+  formatISO,
+  isSameDay,
+  isSameMonth,
+  startOfDay,
+} from 'date-fns';
 
 import {
   alphabetIds,
@@ -188,8 +194,8 @@ export const reportSlice = createSlice({
       }>
     ) => {
       state.dirty = true;
-      state.startDate = action.payload.startDate;
+      state.startDate = formatISO(startOfDay(action.payload.startDate));
-      state.endDate = action.payload.endDate;
+      state.endDate = formatISO(endOfDay(action.payload.endDate));
 
       if (isSameDay(state.startDate, state.endDate)) {
         state.interval = 'hour';
@@ -203,7 +209,7 @@ export const reportSlice = createSlice({
     // Date range
     changeStartDate: (state, action: PayloadAction<string>) => {
       state.dirty = true;
-      state.startDate = action.payload;
+      state.startDate = formatISO(startOfDay(action.payload));
 
       const interval = getDefaultIntervalByDates(
         state.startDate,
@@ -217,7 +223,7 @@ export const reportSlice = createSlice({
     // Date range
     changeEndDate: (state, action: PayloadAction<string>) => {
       state.dirty = true;
-      state.endDate = action.payload;
+      state.endDate = formatISO(endOfDay(action.payload));
 
       const interval = getDefaultIntervalByDates(
         state.startDate,

packages/common/src/date.ts

@@ -5,3 +5,41 @@ export function getTime(date: string | number | Date) {
 export function toISOString(date: string | number | Date) {
   return new Date(date).toISOString();
 }
+
+export function getTimezoneFromDateString(_date: string) {
+  const mapper: Record<string, string> = {
+    '+00:00': 'UTC',
+    '+01:00': 'Europe/Paris',
+    '+02:00': 'Europe/Stockholm',
+    '+03:00': 'Europe/Moscow',
+    '+04:00': 'Asia/Dubai',
+    '+05:00': 'Asia/Karachi',
+    '+06:00': 'Asia/Dhaka',
+    '+07:00': 'Asia/Bangkok',
+    '+08:00': 'Asia/Shanghai',
+    '+09:00': 'Asia/Tokyo',
+    '+10:00': 'Australia/Sydney',
+    '+11:00': 'Pacific/Noumea',
+    '+12:00': 'Pacific/Fiji',
+    '-02:00': 'America/Noronha',
+    '-03:00': 'America/Sao_Paulo',
+    '-04:00': 'America/Santiago',
+    '-05:00': 'America/Bogota',
+    '-06:00': 'America/Mexico_City',
+    '-07:00': 'America/Phoenix',
+    '-08:00': 'America/Los_Angeles',
+    '-09:00': 'America/Anchorage',
+    '-10:00': 'Pacific/Honolulu',
+    '-11:00': 'Pacific/Midway',
+    '-12:00': 'Pacific/Tarawa',
+  };
+
+  const defaultTimezone = 'UTC';
+
+  const match = _date.match(/([+-][0-9]{2}):([0-9]{2})$/)?.[0];
+  if (match) {
+    return mapper[match] ?? defaultTimezone;
+  }
+
+  return defaultTimezone;
+}

packages/constants/index.ts

@@ -15,7 +15,7 @@ export const timeWindows = {
   },
   today: {
     key: 'today',
-    label: 'Today',
+    label: '24 hours',
     shortcut: 'D',
   },
   '7d': {

packages/db/src/clickhouse-client.ts

@@ -15,6 +15,7 @@ export const ch = createClient({
 export async function chQueryWithMeta<T extends Record<string, any>>(
   query: string
 ): Promise<ResponseJSON<T>> {
+  console.log('Query:', query);
   const start = Date.now();
   const res = await ch.query({
     query,
@@ -39,7 +40,6 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
 
   console.log(`Clickhouse query took ${response.statistics?.elapsed}ms`);
   console.log(`chQuery took ${Date.now() - start}ms`);
-  console.log('Query:', query);
 
   return response;
 }
@@ -52,7 +52,11 @@ export async function chQuery<T extends Record<string, any>>(
 
 export function formatClickhouseDate(_date: Date | string) {
   const date = typeof _date === 'string' ? new Date(_date) : _date;
-  return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
+  return date
+    .toISOString()
+    .replace('T', ' ')
+    .replace(/Z+$/, '')
+    .replace(/\.[0-9]+$/, '');
 }
 
 export function convertClickhouseDateToJs(date: string) {

packages/db/src/services/chart.service.ts

@@ -1,5 +1,6 @@
 import { escape } from 'sqlstring';
 
+import { getTimezoneFromDateString } from '@openpanel/common';
 import type {
   IChartEventFilter,
   IGetChartDataInput,
@@ -21,32 +22,40 @@ export function getChartSql({
 
   sb.where = getEventFiltersWhereClause(event.filters);
   sb.where.projectId = `project_id = ${escape(projectId)}`;
+
+  let labelValue = escape('*');
   if (event.name !== '*') {
-    sb.select.label = `${escape(event.name)} as label`;
+    labelValue = `${escape(event.name)}`;
-    sb.where.eventName = `name = ${escape(event.name)}`;
+    sb.select.label = `${labelValue} as label`;
+    sb.where.eventName = `name = ${labelValue}`;
+  } else {
+    sb.select.label = `${labelValue} as label`;
   }
 
   sb.select.count = `count(*) as count`;
   switch (interval) {
     case 'minute': {
-      sb.select.date = `toStartOfMinute(created_at) as date`;
+      sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
     case 'hour': {
-      sb.select.date = `toStartOfHour(created_at) as date`;
+      sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
     case 'day': {
-      sb.select.date = `toStartOfDay(created_at) as date`;
+      sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
     case 'month': {
-      sb.select.date = `toStartOfMonth(created_at) as date`;
+      sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
   }
   sb.groupBy.date = 'date';
-  sb.orderBy.date = 'date ASC';
 
   if (startDate) {
     sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;

packages/trpc/src/routers/chart.helpers.ts

@@ -1,10 +1,14 @@
 import {
+  differenceInMilliseconds,
   endOfDay,
+  endOfMonth,
   endOfYear,
+  formatISO,
   startOfDay,
   startOfMonth,
   startOfYear,
   subDays,
+  subMilliseconds,
   subMinutes,
   subMonths,
   subYears,
@@ -43,129 +47,6 @@ function getEventLegend(event: IChartEvent) {
   return event.displayName ?? `${event.name} (${event.id})`;
 }
 
-function fillEmptySpotsInTimeline(
-  items: ResultItem[],
-  interval: IInterval,
-  startDate: string,
-  endDate: string
-) {
-  const result = [];
-  const clonedStartDate = new Date(startDate);
-  const clonedEndDate = new Date(endDate);
-  const today = new Date();
-
-  if (interval === 'minute') {
-    clonedStartDate.setUTCSeconds(0, 0);
-    clonedEndDate.setUTCMinutes(clonedEndDate.getUTCMinutes() + 1, 0, 0);
-  } else if (interval === 'hour') {
-    clonedStartDate.setUTCMinutes(0, 0, 0);
-    clonedEndDate.setUTCMinutes(0, 0, 0);
-  } else {
-    clonedStartDate.setUTCHours(0, 0, 0, 0);
-    clonedEndDate.setUTCHours(0, 0, 0, 0);
-  }
-
-  if (interval === 'month') {
-    clonedStartDate.setUTCDate(1);
-    clonedEndDate.setUTCDate(1);
-  }
-
-  // Force if interval is month and the start date is the same month as today
-  const shouldForce = () =>
-    interval === 'month' &&
-    clonedStartDate.getUTCFullYear() === today.getUTCFullYear() &&
-    clonedStartDate.getUTCMonth() === today.getUTCMonth();
-  let prev = undefined;
-  while (
-    shouldForce() ||
-    clonedStartDate.getTime() <= clonedEndDate.getTime()
-  ) {
-    if (prev === clonedStartDate.getTime()) {
-      break;
-    }
-    prev = clonedStartDate.getTime();
-
-    const getYear = (date: Date) => date.getUTCFullYear();
-    const getMonth = (date: Date) => date.getUTCMonth();
-    const getDay = (date: Date) => date.getUTCDate();
-    const getHour = (date: Date) => date.getUTCHours();
-    const getMinute = (date: Date) => date.getUTCMinutes();
-
-    const item = items.find((item) => {
-      const date = convertClickhouseDateToJs(item.date);
-
-      if (interval === 'month') {
-        return (
-          getYear(date) === getYear(clonedStartDate) &&
-          getMonth(date) === getMonth(clonedStartDate)
-        );
-      }
-      if (interval === 'day') {
-        return (
-          getYear(date) === getYear(clonedStartDate) &&
-          getMonth(date) === getMonth(clonedStartDate) &&
-          getDay(date) === getDay(clonedStartDate)
-        );
-      }
-      if (interval === 'hour') {
-        return (
-          getYear(date) === getYear(clonedStartDate) &&
-          getMonth(date) === getMonth(clonedStartDate) &&
-          getDay(date) === getDay(clonedStartDate) &&
-          getHour(date) === getHour(clonedStartDate)
-        );
-      }
-      if (interval === 'minute') {
-        return (
-          getYear(date) === getYear(clonedStartDate) &&
-          getMonth(date) === getMonth(clonedStartDate) &&
-          getDay(date) === getDay(clonedStartDate) &&
-          getHour(date) === getHour(clonedStartDate) &&
-          getMinute(date) === getMinute(clonedStartDate)
-        );
-      }
-
-      return false;
-    });
-
-    if (item) {
-      result.push({
-        ...item,
-        date: clonedStartDate.toISOString(),
-      });
-    } else {
-      result.push({
-        date: clonedStartDate.toISOString(),
-        count: 0,
-        label: null,
-      });
-    }
-
-    switch (interval) {
-      case 'day': {
-        clonedStartDate.setUTCDate(clonedStartDate.getUTCDate() + 1);
-        break;
-      }
-      case 'hour': {
-        clonedStartDate.setUTCHours(clonedStartDate.getUTCHours() + 1);
-        break;
-      }
-      case 'minute': {
-        clonedStartDate.setUTCMinutes(clonedStartDate.getUTCMinutes() + 1);
-        break;
-      }
-      case 'month': {
-        clonedStartDate.setUTCMonth(clonedStartDate.getUTCMonth() + 1);
-        break;
-      }
-    }
-  }
-
-  return sort(function (a, b) {
-    return new Date(a.date).getTime() - new Date(b.date).getTime();
-  }, result);
-}
-
 export function withFormula(
   { formula, events }: IChartInput,
   series: GetChartDataResult
@@ -235,6 +116,26 @@ export function withFormula(
   ];
 }
 
+const toDynamicISODateWithTZ = (
+  date: string,
+  blueprint: string,
+  interval: IInterval
+) => {
+  // If we have a space in the date we know it's a date with time
+  if (date.includes(' ')) {
+    // If interval is minutes we need to convert the timezone to what timezone is used (either on client or the server)
+    // - We use timezone from server if its a predefined range (yearToDate, lastYear, etc.)
+    // - We use timezone from client if its a custom range
+    if (interval === 'minute' || interval === 'hour') {
+      return date.replace(' ', 'T') + blueprint.slice(-6);
+    }
+    // Otherwise we just return without the timezone
+    // It will be converted to the correct timezone on the client
+    return date.replace(' ', 'T');
+  }
+  return `${date}T00:00:00`;
+};
+
 export async function getChartData(payload: IGetChartDataInput) {
   let result = await chQuery<ResultItem>(getChartSql(payload));
 
@@ -250,9 +151,15 @@ export async function getChartData(payload: IGetChartDataInput) {
   // group by sql label
   const series = result.reduce(
     (acc, item) => {
+      // If we fill empty spots in the timeline (clickhouse) we wont get a label back
+      // take the event name as label
+      if (!item.label && item.count === 0) {
+        item.label = payload.event.name;
+      }
+
+      const label = item.label?.trim() || NOT_SET_VALUE;
       // item.label can be null when using breakdowns on a property
       // that doesn't exist on all events
-      const label = item.label?.trim() || NOT_SET_VALUE;
       if (label) {
         if (acc[label]) {
           acc[label]?.push(item);
@@ -281,22 +188,25 @@ export async function getChartData(payload: IGetChartDataInput) {
       payload.chartType === 'metric' ||
       payload.chartType === 'pie' ||
       payload.chartType === 'bar'
-        ? fillEmptySpotsInTimeline(
+        ? (series[key] ?? []).map((item) => {
-            series[key] ?? [],
-            payload.interval,
-            payload.startDate,
-            payload.endDate
-          ).map((item) => {
             return {
               label: serieName,
               count: item.count ? round(item.count) : null,
-              date: new Date(item.date).toISOString(),
+              date: toDynamicISODateWithTZ(
+                item.date,
+                payload.startDate,
+                payload.interval
+              ),
             };
           })
         : (series[key] ?? []).map((item) => ({
             label: item.label,
             count: item.count ? round(item.count) : null,
-            date: new Date(item.date).toISOString(),
+            date: toDynamicISODateWithTZ(
+              item.date,
+              payload.startDate,
+              payload.interval
+            ),
           }));
 
     return {
@@ -310,8 +220,8 @@ export async function getChartData(payload: IGetChartDataInput) {
 export function getDatesFromRange(range: IChartRange) {
   if (range === '30min' || range === 'lastHour') {
     const minutes = range === '30min' ? 30 : 60;
-    const startDate = subMinutes(new Date(), minutes).toUTCString();
+    const startDate = formatISO(subMinutes(new Date(), minutes));
-    const endDate = new Date().toUTCString();
+    const endDate = formatISO(new Date());
 
     return {
       startDate,
@@ -320,18 +230,22 @@ export function getDatesFromRange(range: IChartRange) {
   }
 
   if (range === 'today') {
-    const startDate = startOfDay(new Date());
+    // This is last 24 hours instead
-    const endDate = endOfDay(new Date());
+    // Makes it easier to handle timezones
+    // const startDate = startOfDay(new Date());
+    // const endDate = endOfDay(new Date());
+    const startDate = subDays(new Date(), 1);
+    const endDate = new Date();
 
     return {
-      startDate: startDate.toUTCString(),
+      startDate: formatISO(startDate),
-      endDate: endDate.toUTCString(),
+      endDate: formatISO(endDate),
     };
   }
 
   if (range === '7d') {
-    const startDate = subDays(new Date(), 7).toUTCString();
+    const startDate = formatISO(subDays(new Date(), 7));
-    const endDate = new Date().toUTCString();
+    const endDate = formatISO(new Date());
 
     return {
       startDate,
@@ -340,8 +254,8 @@ export function getDatesFromRange(range: IChartRange) {
   }
 
   if (range === '30d') {
-    const startDate = subDays(new Date(), 30).toUTCString();
+    const startDate = formatISO(subDays(new Date(), 30));
-    const endDate = new Date().toUTCString();
+    const endDate = formatISO(new Date());
 
     return {
       startDate,
@@ -350,8 +264,8 @@ export function getDatesFromRange(range: IChartRange) {
   }
 
   if (range === 'monthToDate') {
-    const startDate = startOfMonth(new Date()).toUTCString();
+    const startDate = formatISO(startOfMonth(new Date()));
-    const endDate = new Date().toUTCString();
+    const endDate = formatISO(new Date());
 
     return {
       startDate,
@@ -361,8 +275,8 @@ export function getDatesFromRange(range: IChartRange) {
 
   if (range === 'lastMonth') {
     const month = subMonths(new Date(), 1);
-    const startDate = startOfMonth(month).toUTCString();
+    const startDate = formatISO(startOfMonth(month));
-    const endDate = endOfDay(month).toUTCString();
+    const endDate = formatISO(endOfMonth(month));
 
     return {
       startDate,
@@ -371,8 +285,8 @@ export function getDatesFromRange(range: IChartRange) {
   }
 
   if (range === 'yearToDate') {
-    const startDate = startOfYear(new Date()).toUTCString();
+    const startDate = formatISO(startOfYear(new Date()));
-    const endDate = new Date().toUTCString();
+    const endDate = formatISO(new Date());
 
     return {
       startDate,
@@ -382,8 +296,8 @@ export function getDatesFromRange(range: IChartRange) {
 
   if (range === 'lastYear') {
     const year = subYears(new Date(), 1);
-    const startDate = startOfYear(year).toUTCString();
+    const startDate = formatISO(startOfYear(year));
-    const endDate = endOfYear(year).toUTCString();
+    const endDate = formatISO(endOfYear(year));
 
     return {
       startDate,
@@ -391,20 +305,9 @@ export function getDatesFromRange(range: IChartRange) {
     };
   }
 
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
-  console.log('-------------------------------');
   return {
-    startDate: subDays(new Date(), 30).toISOString(),
+    startDate: formatISO(subDays(new Date(), 30)),
-    endDate: new Date().toISOString(),
+    endDate: formatISO(new Date()),
   };
 }
 
@@ -421,16 +324,15 @@ export function getChartStartEndDate({
 export function getChartPrevStartEndDate({
   startDate,
   endDate,
-  range,
 }: {
   startDate: string;
   endDate: string;
   range: IChartRange;
 }) {
-  const diff = new Date(endDate).getTime() - new Date(startDate).getTime();
+  const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate));
   return {
-    startDate: new Date(new Date(startDate).getTime() - diff).toISOString(),
+    startDate: formatISO(subMilliseconds(new Date(startDate), diff - 1)),
-    endDate: new Date(new Date(endDate).getTime() - diff).toISOString(),
+    endDate: formatISO(subMilliseconds(new Date(endDate), diff - 1)),
   };
 }