fix timezones

packages/common/index.ts

@@ -6,3 +6,4 @@ export * from './src/names';
 export * from './src/string';
 export * from './src/math';
 export * from './src/slug';
+export * from './src/fill-series';

packages/common/package.json

@@ -8,6 +8,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "date-fns": "^3.3.1",
     "mathjs": "^12.3.2",
     "ramda": "^0.29.1",
     "slugify": "^1.6.6",
@@ -15,6 +16,7 @@
     "unique-names-generator": "^4.7.1"
   },
   "devDependencies": {
+    "@openpanel/validation": "workspace:*",
     "@openpanel/eslint-config": "workspace:*",
     "@openpanel/prettier-config": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",

packages/common/src/fill-series.ts

@@ -0,0 +1,93 @@
+import {
+  addDays,
+  addHours,
+  addMinutes,
+  addMonths,
+  format,
+  parseISO,
+  startOfDay,
+  startOfHour,
+  startOfMinute,
+  startOfMonth,
+} from 'date-fns';
+
+import type { IInterval } from '@openpanel/validation';
+
+// Define the data structure
+interface DataEntry {
+  label: string;
+  count: number | null;
+  date: string;
+}
+
+// Function to round down the date to the nearest interval
+function roundDate(date: Date, interval: IInterval): Date {
+  switch (interval) {
+    case 'minute':
+      return startOfMinute(date);
+    case 'hour':
+      return startOfHour(date);
+    case 'day':
+      return startOfDay(date);
+    case 'month':
+      return startOfMonth(date);
+    default:
+      return startOfMinute(date);
+  }
+}
+
+// Function to complete the timeline for each label
+export function completeTimeline(
+  data: DataEntry[],
+  _startDate: string,
+  _endDate: string,
+  interval: IInterval
+) {
+  const startDate = parseISO(_startDate);
+  const endDate = parseISO(_endDate);
+  // Group data by label
+  const labelsMap = new Map<string, Map<string, number>>();
+  data.forEach((entry) => {
+    const roundedDate = roundDate(parseISO(entry.date), interval);
+    const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss');
+
+    if (!labelsMap.has(entry.label)) {
+      labelsMap.set(entry.label, new Map());
+    }
+    const labelData = labelsMap.get(entry.label);
+    labelData?.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0));
+  });
+
+  // Complete the timeline for each label
+  const result: Record<string, DataEntry[]> = {};
+  labelsMap.forEach((counts, label) => {
+    let currentDate = roundDate(startDate, interval);
+    result[label] = [];
+    while (currentDate <= endDate) {
+      const dateKey = format(currentDate, 'yyyy-MM-dd HH:mm:ss');
+      result[label]!.push({
+        label: label,
+        date: dateKey,
+        count: counts.get(dateKey) || 0,
+      });
+
+      // Increment the current date based on the interval
+      switch (interval) {
+        case 'minute':
+          currentDate = addMinutes(currentDate, 1);
+          break;
+        case 'hour':
+          currentDate = addHours(currentDate, 1);
+          break;
+        case 'day':
+          currentDate = addDays(currentDate, 1);
+          break;
+        case 'month':
+          currentDate = addMonths(currentDate, 1);
+          break;
+      }
+    }
+  });
+
+  return result;
+}

packages/db/src/services/chart.service.ts

@@ -23,35 +23,29 @@ export function getChartSql({
   sb.where = getEventFiltersWhereClause(event.filters);
   sb.where.projectId = `project_id = ${escape(projectId)}`;
 
-  let labelValue = escape('*');
   if (event.name !== '*') {
-    labelValue = `${escape(event.name)}`;
+    sb.select.label = `${escape(event.name)} as label`;
-    sb.select.label = `${labelValue} as label`;
+    sb.where.eventName = `name = ${escape(event.name)}`;
-    sb.where.eventName = `name = ${labelValue}`;
   } else {
-    sb.select.label = `${labelValue} as label`;
+    sb.select.label = `'*' as label`;
   }
 
   sb.select.count = `count(*) as count`;
   switch (interval) {
     case 'minute': {
       sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
-      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
     case 'hour': {
       sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
-      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
     case 'day': {
       sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
-      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
     case 'month': {
       sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
-      sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;
       break;
     }
   }

packages/trpc/src/routers/chart.helpers.ts

@@ -17,7 +17,7 @@ import * as mathjs from 'mathjs';
 import { repeat, reverse, sort } from 'ramda';
 import { escape } from 'sqlstring';
 
-import { round } from '@openpanel/common';
+import { completeTimeline, round } from '@openpanel/common';
 import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
 import {
   chQuery,
@@ -133,88 +133,59 @@ const toDynamicISODateWithTZ = (
     // It will be converted to the correct timezone on the client
     return date.replace(' ', 'T');
   }
-  return `${date}T00:00:00`;
+  return `${date}T00:00:00Z`;
 };
 
 export async function getChartData(payload: IGetChartDataInput) {
-  let result = await chQuery<ResultItem>(getChartSql(payload));
+  async function getSeries() {
-
+    const result = await chQuery<ResultItem>(getChartSql(payload));
-  if (result.length === 0 && payload.breakdowns.length > 0) {
+    if (result.length === 0 && payload.breakdowns.length > 0) {
-    result = await chQuery<ResultItem>(
+      return await chQuery<ResultItem>(
-      getChartSql({
+        getChartSql({
-        ...payload,
+          ...payload,
-        breakdowns: [],
+          breakdowns: [],
-      })
+        })
-    );
+      );
+    }
+    return result;
   }
 
-  // group by sql label
+  return getSeries()
-  const series = result.reduce(
+    .then((data) =>
-    (acc, item) => {
+      completeTimeline(
-      // If we fill empty spots in the timeline (clickhouse) we wont get a label back
+        data.map((item) => {
-      // take the event name as label
+          const label = item.label?.trim() || NOT_SET_VALUE;
-      if (!item.label && item.count === 0) {
-        item.label = payload.event.name;
-      }
 
-      const label = item.label?.trim() || NOT_SET_VALUE;
+          return {
-      // item.label can be null when using breakdowns on a property
+            ...item,
-      // that doesn't exist on all events
-      if (label) {
-        if (acc[label]) {
-          acc[label]?.push(item);
-        } else {
-          acc[label] = [item];
-        }
-      }
-
-      return {
-        ...acc,
-      };
-    },
-    {} as Record<string, ResultItem[]>
-  );
-
-  return Object.keys(series).map((key) => {
-    // If we have breakdowns, we want to use the breakdown key as the legend
-    // But only if it successfully broke it down, otherwise we use the getEventLabel
-    const isBreakdown =
-      payload.breakdowns.length && !alphabetIds.includes(key as 'A');
-    const serieName = isBreakdown ? key : getEventLegend(payload.event);
-    const data =
-      payload.chartType === 'area' ||
-      payload.chartType === 'linear' ||
-      payload.chartType === 'histogram' ||
-      payload.chartType === 'metric' ||
-      payload.chartType === 'pie' ||
-      payload.chartType === 'bar'
-        ? (series[key] ?? []).map((item) => {
-            return {
-              label: serieName,
-              count: item.count ? round(item.count) : null,
-              date: toDynamicISODateWithTZ(
-                item.date,
-                payload.startDate,
-                payload.interval
-              ),
-            };
-          })
-        : (series[key] ?? []).map((item) => ({
-            label: item.label,
             count: item.count ? round(item.count) : null,
+            label,
+          };
+        }),
+        payload.startDate,
+        payload.endDate,
+        payload.interval
+      )
+    )
+    .then((series) => {
+      return Object.keys(series).map((label) => {
+        const isBreakdown =
+          payload.breakdowns.length && !alphabetIds.includes(label as 'A');
+        const serieLabel = isBreakdown ? label : getEventLegend(payload.event);
+        return {
+          name: serieLabel,
+          event: payload.event,
+          data: series[label]!.map((item) => ({
+            ...item,
             date: toDynamicISODateWithTZ(
               item.date,
               payload.startDate,
               payload.interval
             ),
-          }));
+          })),
-
+        };
-    return {
+      });
-      name: serieName,
+    });
-      event: payload.event,
-      data,
-    };
-  });
 }
 
 export function getDatesFromRange(range: IChartRange) {

packages/trpc/src/routers/chart.ts

@@ -184,7 +184,7 @@ export const chartRouter = createTRPCRouter({
     }),
 
   // TODO: Make this private
-  chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => {
+  chart: publicProcedure.input(zChartInput).query(async ({ input }) => {
     const currentPeriod = getChartStartEndDate(input);
     const previousPeriod = getChartPrevStartEndDate({
       range: input.range,

pnpm-lock.yaml

@@ -763,6 +763,9 @@ importers:
 
   packages/common:
     dependencies:
+      date-fns:
+        specifier: ^3.3.1
+        version: 3.3.1
       mathjs:
         specifier: ^12.3.2
         version: 12.3.2
@@ -788,6 +791,9 @@ importers:
       '@openpanel/tsconfig':
         specifier: workspace:*
         version: link:../../tooling/typescript
+      '@openpanel/validation':
+        specifier: workspace:*
+        version: link:../validation
       '@types/node':
         specifier: ^18.16.0
         version: 18.19.17