reduce chart payload

packages/common/package.json

@@ -8,6 +8,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@openpanel/constants": "workspace:*",
     "date-fns": "^3.3.1",
     "mathjs": "^12.3.2",
     "ramda": "^0.29.1",
@@ -34,4 +35,4 @@
     ]
   },
   "prettier": "@openpanel/prettier-config"
-}
+}

packages/common/src/fill-series.ts

@@ -11,12 +11,13 @@ import {
   startOfMonth,
 } from 'date-fns';
 
+import { NOT_SET_VALUE } from '@openpanel/constants';
 import type { IInterval } from '@openpanel/validation';
 
 // Define the data structure
-interface DataEntry {
-  label: string;
-  count: number | null;
+export interface ISerieDataItem {
+  label: string | null | undefined;
+  count: number;
   date: string;
 }
 
@@ -37,8 +38,8 @@ function roundDate(date: Date, interval: IInterval): Date {
 }
 
 // Function to complete the timeline for each label
-export function completeTimeline(
-  data: DataEntry[],
+export function completeSerie(
+  data: ISerieDataItem[],
   _startDate: string,
   _endDate: string,
   interval: IInterval
@@ -50,16 +51,16 @@ export function completeTimeline(
   data.forEach((entry) => {
     const roundedDate = roundDate(parseISO(entry.date), interval);
     const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss');
-
-    if (!labelsMap.has(entry.label)) {
-      labelsMap.set(entry.label, new Map());
+    const label = entry.label || NOT_SET_VALUE;
+    if (!labelsMap.has(label)) {
+      labelsMap.set(label, new Map());
     }
-    const labelData = labelsMap.get(entry.label);
+    const labelData = labelsMap.get(label);
     labelData?.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0));
   });
 
   // Complete the timeline for each label
-  const result: Record<string, DataEntry[]> = {};
+  const result: Record<string, ISerieDataItem[]> = {};
   labelsMap.forEach((counts, label) => {
     let currentDate = roundDate(startDate, interval);
     result[label] = [];

packages/common/src/math.ts

@@ -14,7 +14,7 @@ export const average = (arr: (number | null)[]) => {
   return Number.isNaN(avg) ? 0 : avg;
 };
 
-export const sum = (arr: (number | null)[]): number =>
+export const sum = (arr: (number | null | undefined)[]): number =>
   round(arr.filter(isNumber).reduce((acc, item) => acc + item, 0));
 
 export const min = (arr: (number | null)[]): number =>

packages/db/src/services/chart.service.ts

@@ -59,18 +59,18 @@ export function getChartSql({
     sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
   }
 
-  const breakdown = breakdowns[0]!;
-  if (breakdown) {
+  breakdowns.forEach((breakdown, index) => {
+    const key = index === 0 ? 'label' : `label_${index}`;
     const value = breakdown.name.startsWith('properties.')
       ? `mapValues(mapExtractKeyLike(properties, ${escape(
           breakdown.name.replace(/^properties\./, '').replace('.*.', '.%.')
         )}))`
       : escape(breakdown.name);
-    sb.select.label = breakdown.name.startsWith('properties.')
-      ? `arrayElement(${value}, 1) as label`
-      : `${breakdown.name} as label`;
-    sb.groupBy.label = `label`;
-  }
+    sb.select[key] = breakdown.name.startsWith('properties.')
+      ? `arrayElement(${value}, 1) as ${key}`
+      : `${breakdown.name} as ${key}`;
+    sb.groupBy[key] = `${key}`;
+  });
 
   if (event.segment === 'user') {
     sb.select.count = `countDistinct(profile_id) as count`;

packages/db/src/services/organization.service.ts

@@ -24,6 +24,7 @@ export function transformOrganization(org: Organization) {
 export async function getCurrentOrganizations() {
   const session = auth();
   if (!session.userId) return [];
+
   const organizations = await db.organization.findMany({
     where: {
       members: {

packages/trpc/src/routers/chart.helpers.ts

@@ -3,6 +3,7 @@ import {
   endOfMonth,
   endOfYear,
   formatISO,
+  startOfDay,
   startOfMonth,
   startOfYear,
   subDays,
@@ -15,8 +16,17 @@ import * as mathjs from 'mathjs';
 import { repeat, reverse } from 'ramda';
 import { escape } from 'sqlstring';
 
-import { completeTimeline, round } from '@openpanel/common';
-import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
+import {
+  average,
+  completeSerie,
+  max,
+  min,
+  round,
+  slug,
+  sum,
+} from '@openpanel/common';
+import type { ISerieDataItem } from '@openpanel/common';
+import { alphabetIds } from '@openpanel/constants';
 import {
   chQuery,
   createSqlBuilder,
@@ -26,27 +36,23 @@ import {
   getProfiles,
 } from '@openpanel/db';
 import type {
+  FinalChart,
   IChartEvent,
   IChartInput,
+  IChartInputWithDates,
   IChartRange,
   IGetChartDataInput,
   IInterval,
+  PreviousValue,
 } from '@openpanel/validation';
 
-export type GetChartDataResult = Awaited<ReturnType<typeof getChartData>>;
-export interface ResultItem {
-  label: string | null;
-  count: number | null;
-  date: string;
-}
-
 function getEventLegend(event: IChartEvent) {
   return event.displayName ?? event.name;
 }
 
 export function withFormula(
   { formula, events }: IChartInput,
-  series: GetChartDataResult
+  series: Awaited<ReturnType<typeof getChartSerie>>
 ) {
   if (!formula) {
     return series;
@@ -145,58 +151,6 @@ const toDynamicISODateWithTZ = (
   return `${date}T00:00:00Z`;
 };
 
-export async function getChartData(payload: IGetChartDataInput) {
-  async function getSeries() {
-    const result = await chQuery<ResultItem>(getChartSql(payload));
-    if (result.length === 0 && payload.breakdowns.length > 0) {
-      return await chQuery<ResultItem>(
-        getChartSql({
-          ...payload,
-          breakdowns: [],
-        })
-      );
-    }
-    return result;
-  }
-
-  return getSeries()
-    .then((data) =>
-      completeTimeline(
-        data.map((item) => {
-          const label = item.label?.trim() || NOT_SET_VALUE;
-
-          return {
-            ...item,
-            count: item.count ? round(item.count) : null,
-            label,
-          };
-        }),
-        payload.startDate,
-        payload.endDate,
-        payload.interval
-      )
-    )
-    .then((series) => {
-      return Object.keys(series).map((label) => {
-        const isBreakdown =
-          payload.breakdowns.length && !alphabetIds.includes(label as 'A');
-        const serieLabel = isBreakdown ? label : getEventLegend(payload.event);
-        return {
-          name: serieLabel,
-          event: payload.event,
-          data: series[label]!.map((item) => ({
-            ...item,
-            date: toDynamicISODateWithTZ(
-              item.date,
-              payload.startDate,
-              payload.interval
-            ),
-          })),
-        };
-      });
-    });
-}
-
 export function getDatesFromRange(range: IChartRange) {
   if (range === '30min' || range === 'lastHour') {
     const minutes = range === '30min' ? 30 : 60;
@@ -224,17 +178,7 @@ export function getDatesFromRange(range: IChartRange) {
   }
 
   if (range === '7d') {
-    const startDate = formatISO(subDays(new Date(), 7));
-    const endDate = formatISO(new Date());
-
-    return {
-      startDate,
-      endDate,
-    };
-  }
-
-  if (range === '30d') {
-    const startDate = formatISO(subDays(new Date(), 30));
+    const startDate = formatISO(startOfDay(subDays(new Date(), 7)));
     const endDate = formatISO(new Date());
 
     return {
@@ -285,9 +229,13 @@ export function getDatesFromRange(range: IChartRange) {
     };
   }
 
+  // range === '30d'
+  const startDate = formatISO(startOfDay(subDays(new Date(), 30)));
+  const endDate = formatISO(new Date());
+
   return {
-    startDate: formatISO(subDays(new Date(), 30)),
-    endDate: formatISO(new Date()),
+    startDate,
+    endDate,
   };
 }
 
@@ -491,22 +439,51 @@ export async function getFunnelStep({
   return getProfiles(res.map((r) => r.id));
 }
 
-export async function getSeriesFromEvents(input: IChartInput) {
-  const { startDate, endDate } =
-    input.startDate && input.endDate
-      ? {
-          startDate: input.startDate,
-          endDate: input.endDate,
-        }
-      : getDatesFromRange(input.range);
+export async function getChartSerie(payload: IGetChartDataInput) {
+  async function getSeries() {
+    const result = await chQuery<ISerieDataItem>(getChartSql(payload));
+    if (result.length === 0 && payload.breakdowns.length > 0) {
+      return await chQuery<ISerieDataItem>(
+        getChartSql({
+          ...payload,
+          breakdowns: [],
+        })
+      );
+    }
+    return result;
+  }
 
+  return getSeries()
+    .then((data) =>
+      completeSerie(data, payload.startDate, payload.endDate, payload.interval)
+    )
+    .then((series) => {
+      return Object.keys(series).map((label) => {
+        const isBreakdown =
+          payload.breakdowns.length && !alphabetIds.includes(label as 'A');
+        const serieLabel = isBreakdown ? label : getEventLegend(payload.event);
+        return {
+          name: serieLabel,
+          event: payload.event,
+          data: series[label]!.map((item) => ({
+            ...item,
+            date: toDynamicISODateWithTZ(
+              item.date,
+              payload.startDate,
+              payload.interval
+            ),
+          })),
+        };
+      });
+    });
+}
+
+export async function getChartSeries(input: IChartInputWithDates) {
   const series = (
     await Promise.all(
       input.events.map(async (event) =>
-        getChartData({
+        getChartSerie({
           ...input,
-          startDate,
-          endDate,
           event,
         })
       )
@@ -519,3 +496,188 @@ export async function getSeriesFromEvents(input: IChartInput) {
     return series;
   }
 }
+
+export async function getChart(input: IChartInput) {
+  const currentPeriod = getChartStartEndDate(input);
+  const previousPeriod = getChartPrevStartEndDate({
+    range: input.range,
+    ...currentPeriod,
+  });
+
+  const promises = [getChartSeries({ ...input, ...currentPeriod })];
+
+  if (input.previous) {
+    promises.push(
+      getChartSeries({
+        ...input,
+        ...previousPeriod,
+      })
+    );
+  }
+
+  const result = await Promise.all(promises);
+  const series = result[0]!;
+  const previousSeries = result[1];
+  const limit = input.limit || 300;
+  const offset = input.offset || 0;
+  const final: FinalChart = {
+    series: series
+      .slice(offset, limit ? offset + limit : series.length)
+      .map((serie) => {
+        const previousSerie = previousSeries?.find(
+          (item) => item.name === serie.name
+        );
+        const metrics = {
+          sum: sum(serie.data.map((item) => item.count)),
+          average: round(average(serie.data.map((item) => item.count)), 2),
+          min: min(serie.data.map((item) => item.count)),
+          max: max(serie.data.map((item) => item.count)),
+        };
+
+        return {
+          id: slug(serie.name),
+          name: serie.name,
+          event: {
+            id: serie.event.id!,
+            name: serie.event.displayName ?? serie.event.name,
+          },
+          metrics: {
+            ...metrics,
+            ...(input.previous
+              ? {
+                  previous: {
+                    sum: getPreviousMetric(
+                      metrics.sum,
+                      previousSerie
+                        ? sum(previousSerie?.data.map((item) => item.count))
+                        : null
+                    ),
+                    average: getPreviousMetric(
+                      metrics.average,
+                      previousSerie
+                        ? round(
+                            average(
+                              previousSerie?.data.map((item) => item.count)
+                            ),
+                            2
+                          )
+                        : null
+                    ),
+                    min: getPreviousMetric(
+                      metrics.sum,
+                      previousSerie
+                        ? min(previousSerie?.data.map((item) => item.count))
+                        : null
+                    ),
+                    max: getPreviousMetric(
+                      metrics.sum,
+                      previousSerie
+                        ? max(previousSerie?.data.map((item) => item.count))
+                        : null
+                    ),
+                  },
+                }
+              : {}),
+          },
+          data: serie.data.map((item, index) => ({
+            date: item.date,
+            count: item.count ?? 0,
+            previous: previousSerie?.data[index]
+              ? getPreviousMetric(
+                  item.count ?? 0,
+                  previousSerie?.data[index]?.count ?? null
+                )
+              : undefined,
+          })),
+        };
+      }),
+    metrics: {
+      sum: 0,
+      average: 0,
+      min: 0,
+      max: 0,
+    },
+  };
+
+  final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
+  final.metrics.average = round(
+    average(final.series.map((item) => item.metrics.average)),
+    2
+  );
+  final.metrics.min = min(final.series.map((item) => item.metrics.min));
+  final.metrics.max = max(final.series.map((item) => item.metrics.max));
+  if (input.previous) {
+    final.metrics.previous = {
+      sum: getPreviousMetric(
+        final.metrics.sum,
+        sum(final.series.map((item) => item.metrics.previous?.sum?.value ?? 0))
+      ),
+      average: getPreviousMetric(
+        final.metrics.average,
+        round(
+          average(
+            final.series.map(
+              (item) => item.metrics.previous?.average?.value ?? 0
+            )
+          ),
+          2
+        )
+      ),
+      min: getPreviousMetric(
+        final.metrics.min,
+        min(final.series.map((item) => item.metrics.previous?.min?.value ?? 0))
+      ),
+      max: getPreviousMetric(
+        final.metrics.max,
+        max(final.series.map((item) => item.metrics.previous?.max?.value ?? 0))
+      ),
+    };
+  }
+
+  // Sort by sum
+  final.series = final.series.sort((a, b) => {
+    if (input.chartType === 'linear') {
+      const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+      const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+      return sumB - sumA;
+    } else {
+      return b.metrics[input.metric] - a.metrics[input.metric];
+    }
+  });
+
+  return final;
+}
+
+export function getPreviousMetric(
+  current: number,
+  previous: number | null
+): PreviousValue {
+  if (previous === null) {
+    return undefined;
+  }
+
+  const diff = round(
+    ((current > previous
+      ? current / previous
+      : current < previous
+        ? previous / current
+        : 0) -
+      1) *
+      100,
+    1
+  );
+
+  return {
+    diff:
+      Number.isNaN(diff) || !Number.isFinite(diff) || current === previous
+        ? null
+        : diff,
+    state:
+      current > previous
+        ? 'positive'
+        : current < previous
+          ? 'negative'
+          : 'neutral',
+    value: previous,
+  };
+}

packages/trpc/src/routers/chart.ts

@@ -5,57 +5,24 @@ import { z } from 'zod';
 import { average, max, min, round, slug, sum } from '@openpanel/common';
 import { chQuery, createSqlBuilder, db } from '@openpanel/db';
 import { zChartInput } from '@openpanel/validation';
-import type { IChartEvent, IChartInput } from '@openpanel/validation';
+import type {
+  FinalChart,
+  IChartInput,
+  PreviousValue,
+} from '@openpanel/validation';
 
 import { getProjectAccessCached } from '../access';
 import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
 import {
+  getChart,
   getChartPrevStartEndDate,
+  getChartSeries,
   getChartStartEndDate,
   getFunnelData,
   getFunnelStep,
-  getSeriesFromEvents,
 } from './chart.helpers';
 
-type PreviousValue = {
-  value: number;
-  diff: number | null;
-  state: 'positive' | 'negative' | 'neutral';
-} | null;
-
-interface Metrics {
-  sum: number;
-  average: number;
-  min: number;
-  max: number;
-  previous: {
-    sum: PreviousValue;
-    average: PreviousValue;
-    min: PreviousValue;
-    max: PreviousValue;
-  };
-}
-
-export interface IChartSerie {
-  id: string;
-  name: string;
-  event: IChartEvent;
-  metrics: Metrics;
-  data: {
-    date: string;
-    count: number;
-    label: string | null;
-    previous: PreviousValue;
-  }[];
-}
-
-export interface FinalChart {
-  events: IChartInput['events'];
-  series: IChartSerie[];
-  metrics: Metrics;
-}
-
 export const chartRouter = createTRPCRouter({
   events: protectedProcedure
     .input(z.object({ projectId: z.string() }))
@@ -91,6 +58,7 @@ export const chartRouter = createTRPCRouter({
         'has_profile',
         'name',
         'path',
+        'origin',
         'referrer',
         'referrer_name',
         'duration',
@@ -208,183 +176,3 @@ export const chartRouter = createTRPCRouter({
     return getChart(input);
   }),
 });
-
-export async function getChart(input: IChartInput) {
-  const currentPeriod = getChartStartEndDate(input);
-  const previousPeriod = getChartPrevStartEndDate({
-    range: input.range,
-    ...currentPeriod,
-  });
-
-  const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];
-
-  if (input.previous) {
-    promises.push(
-      getSeriesFromEvents({
-        ...input,
-        ...previousPeriod,
-      })
-    );
-  }
-
-  const result = await Promise.all(promises);
-  const series = result[0]!;
-  const previousSeries = result[1];
-
-  const final: FinalChart = {
-    events: input.events,
-    series: series.map((serie) => {
-      const previousSerie = previousSeries?.find(
-        (item) => item.name === serie.name
-      );
-      const metrics = {
-        sum: sum(serie.data.map((item) => item.count)),
-        average: round(average(serie.data.map((item) => item.count)), 2),
-        min: min(serie.data.map((item) => item.count)),
-        max: max(serie.data.map((item) => item.count)),
-      };
-
-      return {
-        id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend
-        name: serie.name,
-        event: {
-          ...serie.event,
-          displayName: serie.event.displayName ?? serie.event.name,
-        },
-        metrics: {
-          ...metrics,
-          previous: {
-            sum: getPreviousMetric(
-              metrics.sum,
-              previousSerie
-                ? sum(previousSerie?.data.map((item) => item.count))
-                : null
-            ),
-            average: getPreviousMetric(
-              metrics.average,
-              previousSerie
-                ? round(
-                    average(previousSerie?.data.map((item) => item.count)),
-                    2
-                  )
-                : null
-            ),
-            min: getPreviousMetric(
-              metrics.sum,
-              previousSerie
-                ? min(previousSerie?.data.map((item) => item.count))
-                : null
-            ),
-            max: getPreviousMetric(
-              metrics.sum,
-              previousSerie
-                ? max(previousSerie?.data.map((item) => item.count))
-                : null
-            ),
-          },
-        },
-        data: serie.data.map((item, index) => ({
-          date: item.date,
-          count: item.count ?? 0,
-          label: item.label,
-          previous: previousSerie?.data[index]
-            ? getPreviousMetric(
-                item.count ?? 0,
-                previousSerie?.data[index]?.count ?? null
-              )
-            : null,
-        })),
-      };
-    }),
-    metrics: {
-      sum: 0,
-      average: 0,
-      min: 0,
-      max: 0,
-      previous: {
-        sum: null,
-        average: null,
-        min: null,
-        max: null,
-      },
-    },
-  };
-
-  final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
-  final.metrics.average = round(
-    average(final.series.map((item) => item.metrics.average)),
-    2
-  );
-  final.metrics.min = min(final.series.map((item) => item.metrics.min));
-  final.metrics.max = max(final.series.map((item) => item.metrics.max));
-  final.metrics.previous = {
-    sum: getPreviousMetric(
-      final.metrics.sum,
-      sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))
-    ),
-    average: getPreviousMetric(
-      final.metrics.average,
-      round(
-        average(
-          final.series.map((item) => item.metrics.previous.average?.value ?? 0)
-        ),
-        2
-      )
-    ),
-    min: getPreviousMetric(
-      final.metrics.min,
-      min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))
-    ),
-    max: getPreviousMetric(
-      final.metrics.max,
-      max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))
-    ),
-  };
-
-  // Sort by sum
-  final.series = final.series.sort((a, b) => {
-    if (input.chartType === 'linear') {
-      const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
-      const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
-      return sumB - sumA;
-    } else {
-      return b.metrics[input.metric] - a.metrics[input.metric];
-    }
-  });
-
-  return final;
-}
-
-export function getPreviousMetric(
-  current: number,
-  previous: number | null
-): PreviousValue {
-  if (previous === null) {
-    return null;
-  }
-
-  const diff = round(
-    ((current > previous
-      ? current / previous
-      : current < previous
-        ? previous / current
-        : 0) -
-      1) *
-      100,
-    1
-  );
-
-  return {
-    diff:
-      Number.isNaN(diff) || !Number.isFinite(diff) || current === previous
-        ? null
-        : diff,
-    state:
-      current > previous
-        ? 'positive'
-        : current < previous
-          ? 'negative'
-          : 'neutral',
-    value: previous,
-  };
-}

packages/validation/src/index.ts

@@ -71,6 +71,8 @@ export const zChartInput = z.object({
   projectId: z.string(),
   startDate: z.string().nullish(),
   endDate: z.string().nullish(),
+  limit: z.number().optional(),
+  offset: z.number().optional(),
 });
 
 export const zReportInput = zChartInput.extend({

packages/validation/src/types.validation.ts

@@ -31,9 +31,54 @@ export type IChartType = z.infer<typeof zChartType>;
 export type IChartMetric = z.infer<typeof zMetric>;
 export type IChartLineType = z.infer<typeof zLineType>;
 export type IChartRange = z.infer<typeof zRange>;
+export interface IChartInputWithDates extends IChartInput {
+  startDate: string;
+  endDate: string;
+}
 export type IGetChartDataInput = {
   event: IChartEvent;
   projectId: string;
   startDate: string;
   endDate: string;
 } & Omit<IChartInput, 'events' | 'name' | 'startDate' | 'endDate' | 'range'>;
+
+export type PreviousValue =
+  | {
+      value: number;
+      diff: number | null;
+      state: 'positive' | 'negative' | 'neutral';
+    }
+  | undefined;
+
+export type Metrics = {
+  sum: number;
+  average: number;
+  min: number;
+  max: number;
+  previous?: {
+    sum: PreviousValue;
+    average: PreviousValue;
+    min: PreviousValue;
+    max: PreviousValue;
+  };
+};
+
+export type IChartSerie = {
+  id: string;
+  name: string;
+  event: {
+    id: string;
+    name: string;
+  };
+  metrics: Metrics;
+  data: {
+    date: string;
+    count: number;
+    previous: PreviousValue;
+  }[];
+};
+
+export type FinalChart = {
+  series: IChartSerie[];
+  metrics: Metrics;
+};