reduce chart payload

apps/api/src/controllers/export.controller.ts

@@ -4,7 +4,7 @@ import { z } from 'zod';
 
 import type { GetEventListOptions } from '@openpanel/db';
 import { ClientType, db, getEventList, getEventsCount } from '@openpanel/db';
-import { getChart } from '@openpanel/trpc/src/routers/chart';
+import { getChart } from '@openpanel/trpc/src/routers/chart.helpers';
 import { zChartInput } from '@openpanel/validation';
 
 async function getProjectId(

apps/dashboard/src/components/overview/overview-top-devices.tsx

@@ -30,6 +30,7 @@ export default function OverviewTopDevices({
       title: 'Top devices',
       btn: 'Devices',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -60,6 +61,7 @@ export default function OverviewTopDevices({
       title: 'Top browser',
       btn: 'Browser',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -90,6 +92,7 @@ export default function OverviewTopDevices({
       title: 'Top Browser Version',
       btn: 'Browser Version',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -120,6 +123,7 @@ export default function OverviewTopDevices({
       title: 'Top OS',
       btn: 'OS',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -150,6 +154,7 @@ export default function OverviewTopDevices({
       title: 'Top OS version',
       btn: 'OS Version',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,

apps/dashboard/src/components/overview/overview-top-events/overview-top-events.tsx

@@ -31,6 +31,7 @@ export default function OverviewTopEvents({
       title: 'Top events',
       btn: 'Your',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -69,6 +70,7 @@ export default function OverviewTopEvents({
       title: 'Top events',
       btn: 'All',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -100,6 +102,7 @@ export default function OverviewTopEvents({
       btn: 'Conversions',
       hide: conversions.length === 0,
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,

apps/dashboard/src/components/overview/overview-top-geo.tsx

@@ -29,6 +29,7 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) {
       title: 'Top countries',
       btn: 'Countries',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -59,6 +60,7 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) {
       title: 'Top regions',
       btn: 'Regions',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -89,6 +91,7 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) {
       title: 'Top cities',
       btn: 'Cities',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,

apps/dashboard/src/components/overview/overview-top-pages.tsx

@@ -28,6 +28,7 @@ export default function OverviewTopPages({ projectId }: OverviewTopPagesProps) {
       title: 'Top pages',
       btn: 'Top pages',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -58,6 +59,7 @@ export default function OverviewTopPages({ projectId }: OverviewTopPagesProps) {
       title: 'Entry Pages',
       btn: 'Entries',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -88,6 +90,7 @@ export default function OverviewTopPages({ projectId }: OverviewTopPagesProps) {
       title: 'Exit Pages',
       btn: 'Exits',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,

apps/dashboard/src/components/overview/overview-top-sources.tsx

@@ -32,6 +32,7 @@ export default function OverviewTopSources({
       title: 'Top sources',
       btn: 'All',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -62,6 +63,7 @@ export default function OverviewTopSources({
       title: 'Top urls',
       btn: 'URLs',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -92,6 +94,7 @@ export default function OverviewTopSources({
       title: 'Top types',
       btn: 'Types',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -122,6 +125,7 @@ export default function OverviewTopSources({
       title: 'UTM Source',
       btn: 'Source',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -152,6 +156,7 @@ export default function OverviewTopSources({
       title: 'UTM Medium',
       btn: 'Medium',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -182,6 +187,7 @@ export default function OverviewTopSources({
       title: 'UTM Campaign',
       btn: 'Campaign',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -212,6 +218,7 @@ export default function OverviewTopSources({
       title: 'UTM Term',
       btn: 'Term',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,
@@ -242,6 +249,7 @@ export default function OverviewTopSources({
       title: 'UTM Content',
       btn: 'Content',
       chart: {
+        limit: 10,
         projectId,
         startDate,
         endDate,

apps/dashboard/src/components/report/chart/Chart.tsx

@@ -20,16 +20,16 @@ export function Chart({
   events,
   breakdowns,
   chartType,
-  name,
   range,
   lineType,
   previous,
   formula,
-  unit,
   metric,
   projectId,
   startDate,
   endDate,
+  limit,
+  offset,
 }: ReportChartProps) {
   const [references] = api.reference.getChartReferences.useSuspenseQuery(
     {
@@ -56,6 +56,8 @@ export function Chart({
       previous,
       formula,
       metric,
+      limit,
+      offset,
     },
     {
       keepPreviousData: true,

apps/dashboard/src/components/report/chart/ChartProvider.tsx

@@ -10,8 +10,7 @@ import {
   useState,
 } from 'react';
 
-import type { IChartSerie } from '@openpanel/trpc/src/routers/chart';
+import type { IChartProps, IChartSerie } from '@openpanel/validation';
-import type { IChartProps } from '@openpanel/validation';
 
 import { ChartLoading } from './ChartLoading';
 import { MetricCardLoading } from './MetricCard';

apps/dashboard/src/components/report/chart/MetricCard.tsx

@@ -44,7 +44,7 @@ export function MetricCard({
     );
   };
 
-  const previous = serie.metrics.previous[metric];
+  const previous = serie.metrics.previous?.[metric];
 
   const graphColors = getDiffIndicator(
     previousIndicatorInverted,
@@ -93,7 +93,7 @@ export function MetricCard({
           <div className="flex min-w-0 items-center gap-2 text-left font-semibold">
             <ColorSquare>{serie.event.id}</ColorSquare>
             <span className="overflow-hidden text-ellipsis whitespace-nowrap text-muted-foreground">
-              {serie.name || serie.event.displayName || serie.event.name}
+              {serie.name}
             </span>
           </div>
           {/* <PreviousDiffIndicator {...serie.metrics.previous[metric]} /> */}
@@ -125,9 +125,9 @@ export function MetricCardEmpty() {
 export function MetricCardLoading() {
   return (
     <div className="flex h-[70px] flex-col justify-between">
-      <div className="bg-def-200 h-4 w-1/2 animate-pulse rounded"></div>
+      <div className="h-4 w-1/2 animate-pulse rounded bg-def-200"></div>
-      <div className="bg-def-200 h-8 w-1/3 animate-pulse rounded"></div>
+      <div className="h-8 w-1/3 animate-pulse rounded bg-def-200"></div>
-      <div className="bg-def-200 h-3 w-1/5 animate-pulse rounded"></div>
+      <div className="h-3 w-1/5 animate-pulse rounded bg-def-200"></div>
     </div>
   );
 }

apps/dashboard/src/components/report/chart/ReportBarChart.tsx

@@ -41,7 +41,7 @@ export function ReportBarChart({ data }: ReportBarChartProps) {
             {...(isClickable ? { onClick: () => onClick(serie) } : {})}
           >
             <div
-              className="bg-def-200 absolute bottom-0 left-0 top-0 rounded"
+              className="absolute bottom-0 left-0 top-0 rounded bg-def-200"
               style={{
                 width: `${(serie.metrics.sum / maxCount) * 100}%`,
               }}
@@ -53,10 +53,10 @@ export function ReportBarChart({ data }: ReportBarChartProps) {
               </div>
               <div className="flex flex-shrink-0 items-center justify-end gap-4">
                 <PreviousDiffIndicatorText
-                  {...serie.metrics.previous[metric]}
+                  {...serie.metrics.previous?.[metric]}
                   className="text-xs font-medium"
                 />
-                {serie.metrics.previous[metric]?.value}
+                {serie.metrics.previous?.[metric]?.value}
                 <div className="text-muted-foreground">
                   {number.format(
                     round((serie.metrics.sum / data.metrics.sum) * 100, 2)

apps/dashboard/src/components/report/chart/ReportChartTooltip.tsx

@@ -51,7 +51,7 @@ export function ReportChartTooltip({
         ) as IRechartPayloadItem;
 
         return (
-          <React.Fragment key={data.label}>
+          <React.Fragment key={data.name}>
             {index === 0 && data.date && (
               <div className="flex justify-between gap-8">
                 <div>{formatDate(new Date(data.date))}</div>
@@ -64,7 +64,7 @@ export function ReportChartTooltip({
               />
               <div className="flex flex-1 flex-col">
                 <div className="min-w-0 max-w-[200px] overflow-hidden text-ellipsis whitespace-nowrap font-medium">
-                  {getLabel(data.label)}
+                  {getLabel(data.name)}
                 </div>
                 <div className="flex justify-between gap-8">
                   <div>{number.formatWithUnit(data.count, unit)}</div>

apps/dashboard/src/components/report/chart/ReportPieChart.tsx

@@ -24,7 +24,7 @@ export function ReportPieChart({ data }: ReportPieChartProps) {
     id: serie.id,
     color: getChartColor(serie.index),
     index: serie.index,
-    label: serie.name,
+    name: serie.name,
     count: serie.metrics.sum,
     percent: serie.metrics.sum / sum,
   }));
@@ -88,7 +88,7 @@ const renderLabel = ({
   innerRadius: number;
   outerRadius: number;
   fill: string;
-  payload: { label: string; percent: number };
+  payload: { name: string; percent: number };
 }) => {
   const RADIAN = Math.PI / 180;
   const radius = 25 + innerRadius + (outerRadius - innerRadius);
@@ -97,7 +97,7 @@ const renderLabel = ({
   const yProcent = cy + radiusProcent * Math.sin(-midAngle * RADIAN);
   const x = cx + radius * Math.cos(-midAngle * RADIAN);
   const y = cy + radius * Math.sin(-midAngle * RADIAN);
-  const label = payload.label;
+  const name = payload.name;
   const percent = round(payload.percent * 100, 1);
 
   return (
@@ -108,7 +108,7 @@ const renderLabel = ({
         fill="white"
         textAnchor="middle"
         dominantBaseline="central"
-        fontSize={10}
+        fontSize={12}
         fontWeight={700}
         pointerEvents={'none'}
       >
@@ -120,9 +120,9 @@ const renderLabel = ({
         fill={fill}
         textAnchor={x > cx ? 'start' : 'end'}
         dominantBaseline="central"
-        fontSize={10}
+        fontSize={12}
       >
-        {truncate(label, 20)}
+        {truncate(name, 20)}
       </text>
     </>
   );

apps/dashboard/src/components/report/chart/ReportTable.tsx

@@ -127,7 +127,7 @@ export function ReportTable({
                       <div className="flex items-center gap-2 font-medium">
                         {number.format(serie.metrics.sum)}
                         <PreviousDiffIndicator
-                          {...serie.metrics.previous.sum}
+                          {...serie.metrics.previous?.sum}
                         />
                       </div>
                     </TableCell>
@@ -135,7 +135,7 @@ export function ReportTable({
                       <div className="flex items-center gap-2 font-medium">
                         {number.format(serie.metrics.average)}
                         <PreviousDiffIndicator
-                          {...serie.metrics.previous.average}
+                          {...serie.metrics.previous?.average}
                         />
                       </div>
                     </TableCell>

apps/dashboard/src/components/report/reportSlice.ts

@@ -51,6 +51,7 @@ const initialState: InitialState = {
   formula: undefined,
   unit: undefined,
   metric: 'sum',
+  limit: 500,
 };
 
 export const reportSlice = createSlice({

apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx

@@ -43,7 +43,7 @@ export function ReportBreakdowns() {
       <div className="flex flex-col gap-4">
         {selectedBreakdowns.map((item, index) => {
           return (
-            <div key={item.name} className="bg-def-100 rounded-lg border">
+            <div key={item.name} className="rounded-lg border bg-def-100">
               <div className="flex items-center gap-2 p-2 px-4">
                 <ColorSquare>{index}</ColorSquare>
                 <Combobox
@@ -68,22 +68,20 @@ export function ReportBreakdowns() {
           );
         })}
 
-        {selectedBreakdowns.length === 0 && (
+        <Combobox
-          <Combobox
+          icon={SplitIcon}
-            icon={SplitIcon}
+          searchable
-            searchable
+          value={''}
-            value={''}
+          onChange={(value) => {
-            onChange={(value) => {
+            dispatch(
-              dispatch(
+              addBreakdown({
-                addBreakdown({
+                name: value,
-                  name: value,
+              })
-                })
+            );
-              );
+          }}
-            }}
+          items={propertiesCombobox}
-            items={propertiesCombobox}
+          placeholder="Select breakdown"
-            placeholder="Select breakdown"
+        />
-          />
-        )}
       </div>
     </div>
   );

apps/dashboard/src/hooks/useRechartDataModel.ts

@@ -1,10 +1,22 @@
 'use client';
 
 import { useMemo } from 'react';
-import type { IChartData, IChartSerieDataItem } from '@/trpc/client';
+import type { IChartData } from '@/trpc/client';
 import { getChartColor } from '@/utils/theme';
 
-export type IRechartPayloadItem = IChartSerieDataItem & { color: string };
+export type IRechartPayloadItem = {
+  id: string;
+  name: string;
+  color: string;
+  event: { id: string; name: string };
+  count: number;
+  date: string;
+  previous?: {
+    value: number;
+    diff: number | null;
+    state: 'positive' | 'negative' | 'neutral';
+  };
+};
 
 export function useRechartDataModel(series: IChartData['series']) {
   return useMemo(() => {
@@ -25,6 +37,9 @@ export function useRechartDataModel(series: IChartData['series']) {
                     acc2[`${serie.id}:count`] = item.count;
                     acc2[`${serie.id}:payload`] = {
                       ...item,
+                      id: serie.id,
+                      event: serie.event,
+                      name: serie.name,
                       color: getChartColor(idx),
                     } satisfies IRechartPayloadItem;
                   }

packages/common/package.json

@@ -8,6 +8,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@openpanel/constants": "workspace:*",
     "date-fns": "^3.3.1",
     "mathjs": "^12.3.2",
     "ramda": "^0.29.1",
@@ -34,4 +35,4 @@
     ]
   },
   "prettier": "@openpanel/prettier-config"
-}
+}

packages/common/src/fill-series.ts

@@ -11,12 +11,13 @@ import {
   startOfMonth,
 } from 'date-fns';
 
+import { NOT_SET_VALUE } from '@openpanel/constants';
 import type { IInterval } from '@openpanel/validation';
 
 // Define the data structure
-interface DataEntry {
+export interface ISerieDataItem {
-  label: string;
+  label: string | null | undefined;
-  count: number | null;
+  count: number;
   date: string;
 }
 
@@ -37,8 +38,8 @@ function roundDate(date: Date, interval: IInterval): Date {
 }
 
 // Function to complete the timeline for each label
-export function completeTimeline(
+export function completeSerie(
-  data: DataEntry[],
+  data: ISerieDataItem[],
   _startDate: string,
   _endDate: string,
   interval: IInterval
@@ -50,16 +51,16 @@ export function completeTimeline(
   data.forEach((entry) => {
     const roundedDate = roundDate(parseISO(entry.date), interval);
     const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss');
-
+    const label = entry.label || NOT_SET_VALUE;
-    if (!labelsMap.has(entry.label)) {
+    if (!labelsMap.has(label)) {
-      labelsMap.set(entry.label, new Map());
+      labelsMap.set(label, new Map());
     }
-    const labelData = labelsMap.get(entry.label);
+    const labelData = labelsMap.get(label);
     labelData?.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0));
   });
 
   // Complete the timeline for each label
-  const result: Record<string, DataEntry[]> = {};
+  const result: Record<string, ISerieDataItem[]> = {};
   labelsMap.forEach((counts, label) => {
     let currentDate = roundDate(startDate, interval);
     result[label] = [];

packages/common/src/math.ts

@@ -14,7 +14,7 @@ export const average = (arr: (number | null)[]) => {
   return Number.isNaN(avg) ? 0 : avg;
 };
 
-export const sum = (arr: (number | null)[]): number =>
+export const sum = (arr: (number | null | undefined)[]): number =>
   round(arr.filter(isNumber).reduce((acc, item) => acc + item, 0));
 
 export const min = (arr: (number | null)[]): number =>

packages/db/src/services/chart.service.ts

@@ -59,18 +59,18 @@ export function getChartSql({
     sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
   }
 
-  const breakdown = breakdowns[0]!;
+  breakdowns.forEach((breakdown, index) => {
-  if (breakdown) {
+    const key = index === 0 ? 'label' : `label_${index}`;
     const value = breakdown.name.startsWith('properties.')
       ? `mapValues(mapExtractKeyLike(properties, ${escape(
           breakdown.name.replace(/^properties\./, '').replace('.*.', '.%.')
         )}))`
       : escape(breakdown.name);
-    sb.select.label = breakdown.name.startsWith('properties.')
+    sb.select[key] = breakdown.name.startsWith('properties.')
-      ? `arrayElement(${value}, 1) as label`
+      ? `arrayElement(${value}, 1) as ${key}`
-      : `${breakdown.name} as label`;
+      : `${breakdown.name} as ${key}`;
-    sb.groupBy.label = `label`;
+    sb.groupBy[key] = `${key}`;
-  }
+  });
 
   if (event.segment === 'user') {
     sb.select.count = `countDistinct(profile_id) as count`;

packages/db/src/services/organization.service.ts

@@ -24,6 +24,7 @@ export function transformOrganization(org: Organization) {
 export async function getCurrentOrganizations() {
   const session = auth();
   if (!session.userId) return [];
+
   const organizations = await db.organization.findMany({
     where: {
       members: {

packages/trpc/src/routers/chart.helpers.ts

@@ -3,6 +3,7 @@ import {
   endOfMonth,
   endOfYear,
   formatISO,
+  startOfDay,
   startOfMonth,
   startOfYear,
   subDays,
@@ -15,8 +16,17 @@ import * as mathjs from 'mathjs';
 import { repeat, reverse } from 'ramda';
 import { escape } from 'sqlstring';
 
-import { completeTimeline, round } from '@openpanel/common';
+import {
-import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
+  average,
+  completeSerie,
+  max,
+  min,
+  round,
+  slug,
+  sum,
+} from '@openpanel/common';
+import type { ISerieDataItem } from '@openpanel/common';
+import { alphabetIds } from '@openpanel/constants';
 import {
   chQuery,
   createSqlBuilder,
@@ -26,27 +36,23 @@ import {
   getProfiles,
 } from '@openpanel/db';
 import type {
+  FinalChart,
   IChartEvent,
   IChartInput,
+  IChartInputWithDates,
   IChartRange,
   IGetChartDataInput,
   IInterval,
+  PreviousValue,
 } from '@openpanel/validation';
 
-export type GetChartDataResult = Awaited<ReturnType<typeof getChartData>>;
-export interface ResultItem {
-  label: string | null;
-  count: number | null;
-  date: string;
-}
-
 function getEventLegend(event: IChartEvent) {
   return event.displayName ?? event.name;
 }
 
 export function withFormula(
   { formula, events }: IChartInput,
-  series: GetChartDataResult
+  series: Awaited<ReturnType<typeof getChartSerie>>
 ) {
   if (!formula) {
     return series;
@@ -145,58 +151,6 @@ const toDynamicISODateWithTZ = (
   return `${date}T00:00:00Z`;
 };
 
-export async function getChartData(payload: IGetChartDataInput) {
-  async function getSeries() {
-    const result = await chQuery<ResultItem>(getChartSql(payload));
-    if (result.length === 0 && payload.breakdowns.length > 0) {
-      return await chQuery<ResultItem>(
-        getChartSql({
-          ...payload,
-          breakdowns: [],
-        })
-      );
-    }
-    return result;
-  }
-
-  return getSeries()
-    .then((data) =>
-      completeTimeline(
-        data.map((item) => {
-          const label = item.label?.trim() || NOT_SET_VALUE;
-
-          return {
-            ...item,
-            count: item.count ? round(item.count) : null,
-            label,
-          };
-        }),
-        payload.startDate,
-        payload.endDate,
-        payload.interval
-      )
-    )
-    .then((series) => {
-      return Object.keys(series).map((label) => {
-        const isBreakdown =
-          payload.breakdowns.length && !alphabetIds.includes(label as 'A');
-        const serieLabel = isBreakdown ? label : getEventLegend(payload.event);
-        return {
-          name: serieLabel,
-          event: payload.event,
-          data: series[label]!.map((item) => ({
-            ...item,
-            date: toDynamicISODateWithTZ(
-              item.date,
-              payload.startDate,
-              payload.interval
-            ),
-          })),
-        };
-      });
-    });
-}
-
 export function getDatesFromRange(range: IChartRange) {
   if (range === '30min' || range === 'lastHour') {
     const minutes = range === '30min' ? 30 : 60;
@@ -224,17 +178,7 @@ export function getDatesFromRange(range: IChartRange) {
   }
 
   if (range === '7d') {
-    const startDate = formatISO(subDays(new Date(), 7));
+    const startDate = formatISO(startOfDay(subDays(new Date(), 7)));
-    const endDate = formatISO(new Date());
-
-    return {
-      startDate,
-      endDate,
-    };
-  }
-
-  if (range === '30d') {
-    const startDate = formatISO(subDays(new Date(), 30));
     const endDate = formatISO(new Date());
 
     return {
@@ -285,9 +229,13 @@ export function getDatesFromRange(range: IChartRange) {
     };
   }
 
+  // range === '30d'
+  const startDate = formatISO(startOfDay(subDays(new Date(), 30)));
+  const endDate = formatISO(new Date());
+
   return {
-    startDate: formatISO(subDays(new Date(), 30)),
+    startDate,
-    endDate: formatISO(new Date()),
+    endDate,
   };
 }
 
@@ -491,22 +439,51 @@ export async function getFunnelStep({
   return getProfiles(res.map((r) => r.id));
 }
 
-export async function getSeriesFromEvents(input: IChartInput) {
+export async function getChartSerie(payload: IGetChartDataInput) {
-  const { startDate, endDate } =
+  async function getSeries() {
-    input.startDate && input.endDate
+    const result = await chQuery<ISerieDataItem>(getChartSql(payload));
-      ? {
+    if (result.length === 0 && payload.breakdowns.length > 0) {
-          startDate: input.startDate,
+      return await chQuery<ISerieDataItem>(
-          endDate: input.endDate,
+        getChartSql({
-        }
+          ...payload,
-      : getDatesFromRange(input.range);
+          breakdowns: [],
+        })
+      );
+    }
+    return result;
+  }
 
+  return getSeries()
+    .then((data) =>
+      completeSerie(data, payload.startDate, payload.endDate, payload.interval)
+    )
+    .then((series) => {
+      return Object.keys(series).map((label) => {
+        const isBreakdown =
+          payload.breakdowns.length && !alphabetIds.includes(label as 'A');
+        const serieLabel = isBreakdown ? label : getEventLegend(payload.event);
+        return {
+          name: serieLabel,
+          event: payload.event,
+          data: series[label]!.map((item) => ({
+            ...item,
+            date: toDynamicISODateWithTZ(
+              item.date,
+              payload.startDate,
+              payload.interval
+            ),
+          })),
+        };
+      });
+    });
+}
+
+export async function getChartSeries(input: IChartInputWithDates) {
   const series = (
     await Promise.all(
       input.events.map(async (event) =>
-        getChartData({
+        getChartSerie({
           ...input,
-          startDate,
-          endDate,
           event,
         })
       )
@@ -519,3 +496,188 @@ export async function getSeriesFromEvents(input: IChartInput) {
     return series;
   }
 }
+
+export async function getChart(input: IChartInput) {
+  const currentPeriod = getChartStartEndDate(input);
+  const previousPeriod = getChartPrevStartEndDate({
+    range: input.range,
+    ...currentPeriod,
+  });
+
+  const promises = [getChartSeries({ ...input, ...currentPeriod })];
+
+  if (input.previous) {
+    promises.push(
+      getChartSeries({
+        ...input,
+        ...previousPeriod,
+      })
+    );
+  }
+
+  const result = await Promise.all(promises);
+  const series = result[0]!;
+  const previousSeries = result[1];
+  const limit = input.limit || 300;
+  const offset = input.offset || 0;
+  const final: FinalChart = {
+    series: series
+      .slice(offset, limit ? offset + limit : series.length)
+      .map((serie) => {
+        const previousSerie = previousSeries?.find(
+          (item) => item.name === serie.name
+        );
+        const metrics = {
+          sum: sum(serie.data.map((item) => item.count)),
+          average: round(average(serie.data.map((item) => item.count)), 2),
+          min: min(serie.data.map((item) => item.count)),
+          max: max(serie.data.map((item) => item.count)),
+        };
+
+        return {
+          id: slug(serie.name),
+          name: serie.name,
+          event: {
+            id: serie.event.id!,
+            name: serie.event.displayName ?? serie.event.name,
+          },
+          metrics: {
+            ...metrics,
+            ...(input.previous
+              ? {
+                  previous: {
+                    sum: getPreviousMetric(
+                      metrics.sum,
+                      previousSerie
+                        ? sum(previousSerie?.data.map((item) => item.count))
+                        : null
+                    ),
+                    average: getPreviousMetric(
+                      metrics.average,
+                      previousSerie
+                        ? round(
+                            average(
+                              previousSerie?.data.map((item) => item.count)
+                            ),
+                            2
+                          )
+                        : null
+                    ),
+                    min: getPreviousMetric(
+                      metrics.sum,
+                      previousSerie
+                        ? min(previousSerie?.data.map((item) => item.count))
+                        : null
+                    ),
+                    max: getPreviousMetric(
+                      metrics.sum,
+                      previousSerie
+                        ? max(previousSerie?.data.map((item) => item.count))
+                        : null
+                    ),
+                  },
+                }
+              : {}),
+          },
+          data: serie.data.map((item, index) => ({
+            date: item.date,
+            count: item.count ?? 0,
+            previous: previousSerie?.data[index]
+              ? getPreviousMetric(
+                  item.count ?? 0,
+                  previousSerie?.data[index]?.count ?? null
+                )
+              : undefined,
+          })),
+        };
+      }),
+    metrics: {
+      sum: 0,
+      average: 0,
+      min: 0,
+      max: 0,
+    },
+  };
+
+  final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
+  final.metrics.average = round(
+    average(final.series.map((item) => item.metrics.average)),
+    2
+  );
+  final.metrics.min = min(final.series.map((item) => item.metrics.min));
+  final.metrics.max = max(final.series.map((item) => item.metrics.max));
+  if (input.previous) {
+    final.metrics.previous = {
+      sum: getPreviousMetric(
+        final.metrics.sum,
+        sum(final.series.map((item) => item.metrics.previous?.sum?.value ?? 0))
+      ),
+      average: getPreviousMetric(
+        final.metrics.average,
+        round(
+          average(
+            final.series.map(
+              (item) => item.metrics.previous?.average?.value ?? 0
+            )
+          ),
+          2
+        )
+      ),
+      min: getPreviousMetric(
+        final.metrics.min,
+        min(final.series.map((item) => item.metrics.previous?.min?.value ?? 0))
+      ),
+      max: getPreviousMetric(
+        final.metrics.max,
+        max(final.series.map((item) => item.metrics.previous?.max?.value ?? 0))
+      ),
+    };
+  }
+
+  // Sort by sum
+  final.series = final.series.sort((a, b) => {
+    if (input.chartType === 'linear') {
+      const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+      const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
+      return sumB - sumA;
+    } else {
+      return b.metrics[input.metric] - a.metrics[input.metric];
+    }
+  });
+
+  return final;
+}
+
+export function getPreviousMetric(
+  current: number,
+  previous: number | null
+): PreviousValue {
+  if (previous === null) {
+    return undefined;
+  }
+
+  const diff = round(
+    ((current > previous
+      ? current / previous
+      : current < previous
+        ? previous / current
+        : 0) -
+      1) *
+      100,
+    1
+  );
+
+  return {
+    diff:
+      Number.isNaN(diff) || !Number.isFinite(diff) || current === previous
+        ? null
+        : diff,
+    state:
+      current > previous
+        ? 'positive'
+        : current < previous
+          ? 'negative'
+          : 'neutral',
+    value: previous,
+  };
+}

packages/trpc/src/routers/chart.ts

@@ -5,57 +5,24 @@ import { z } from 'zod';
 import { average, max, min, round, slug, sum } from '@openpanel/common';
 import { chQuery, createSqlBuilder, db } from '@openpanel/db';
 import { zChartInput } from '@openpanel/validation';
-import type { IChartEvent, IChartInput } from '@openpanel/validation';
+import type {
+  FinalChart,
+  IChartInput,
+  PreviousValue,
+} from '@openpanel/validation';
 
 import { getProjectAccessCached } from '../access';
 import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
 import {
+  getChart,
   getChartPrevStartEndDate,
+  getChartSeries,
   getChartStartEndDate,
   getFunnelData,
   getFunnelStep,
-  getSeriesFromEvents,
 } from './chart.helpers';
 
-type PreviousValue = {
-  value: number;
-  diff: number | null;
-  state: 'positive' | 'negative' | 'neutral';
-} | null;
-
-interface Metrics {
-  sum: number;
-  average: number;
-  min: number;
-  max: number;
-  previous: {
-    sum: PreviousValue;
-    average: PreviousValue;
-    min: PreviousValue;
-    max: PreviousValue;
-  };
-}
-
-export interface IChartSerie {
-  id: string;
-  name: string;
-  event: IChartEvent;
-  metrics: Metrics;
-  data: {
-    date: string;
-    count: number;
-    label: string | null;
-    previous: PreviousValue;
-  }[];
-}
-
-export interface FinalChart {
-  events: IChartInput['events'];
-  series: IChartSerie[];
-  metrics: Metrics;
-}
-
 export const chartRouter = createTRPCRouter({
   events: protectedProcedure
     .input(z.object({ projectId: z.string() }))
@@ -91,6 +58,7 @@ export const chartRouter = createTRPCRouter({
         'has_profile',
         'name',
         'path',
+        'origin',
         'referrer',
         'referrer_name',
         'duration',
@@ -208,183 +176,3 @@ export const chartRouter = createTRPCRouter({
     return getChart(input);
   }),
 });
-
-export async function getChart(input: IChartInput) {
-  const currentPeriod = getChartStartEndDate(input);
-  const previousPeriod = getChartPrevStartEndDate({
-    range: input.range,
-    ...currentPeriod,
-  });
-
-  const promises = [getSeriesFromEvents({ ...input, ...currentPeriod })];
-
-  if (input.previous) {
-    promises.push(
-      getSeriesFromEvents({
-        ...input,
-        ...previousPeriod,
-      })
-    );
-  }
-
-  const result = await Promise.all(promises);
-  const series = result[0]!;
-  const previousSeries = result[1];
-
-  const final: FinalChart = {
-    events: input.events,
-    series: series.map((serie) => {
-      const previousSerie = previousSeries?.find(
-        (item) => item.name === serie.name
-      );
-      const metrics = {
-        sum: sum(serie.data.map((item) => item.count)),
-        average: round(average(serie.data.map((item) => item.count)), 2),
-        min: min(serie.data.map((item) => item.count)),
-        max: max(serie.data.map((item) => item.count)),
-      };
-
-      return {
-        id: slug(serie.name), // TODO: Remove this (temporary fix for the frontend
-        name: serie.name,
-        event: {
-          ...serie.event,
-          displayName: serie.event.displayName ?? serie.event.name,
-        },
-        metrics: {
-          ...metrics,
-          previous: {
-            sum: getPreviousMetric(
-              metrics.sum,
-              previousSerie
-                ? sum(previousSerie?.data.map((item) => item.count))
-                : null
-            ),
-            average: getPreviousMetric(
-              metrics.average,
-              previousSerie
-                ? round(
-                    average(previousSerie?.data.map((item) => item.count)),
-                    2
-                  )
-                : null
-            ),
-            min: getPreviousMetric(
-              metrics.sum,
-              previousSerie
-                ? min(previousSerie?.data.map((item) => item.count))
-                : null
-            ),
-            max: getPreviousMetric(
-              metrics.sum,
-              previousSerie
-                ? max(previousSerie?.data.map((item) => item.count))
-                : null
-            ),
-          },
-        },
-        data: serie.data.map((item, index) => ({
-          date: item.date,
-          count: item.count ?? 0,
-          label: item.label,
-          previous: previousSerie?.data[index]
-            ? getPreviousMetric(
-                item.count ?? 0,
-                previousSerie?.data[index]?.count ?? null
-              )
-            : null,
-        })),
-      };
-    }),
-    metrics: {
-      sum: 0,
-      average: 0,
-      min: 0,
-      max: 0,
-      previous: {
-        sum: null,
-        average: null,
-        min: null,
-        max: null,
-      },
-    },
-  };
-
-  final.metrics.sum = sum(final.series.map((item) => item.metrics.sum));
-  final.metrics.average = round(
-    average(final.series.map((item) => item.metrics.average)),
-    2
-  );
-  final.metrics.min = min(final.series.map((item) => item.metrics.min));
-  final.metrics.max = max(final.series.map((item) => item.metrics.max));
-  final.metrics.previous = {
-    sum: getPreviousMetric(
-      final.metrics.sum,
-      sum(final.series.map((item) => item.metrics.previous.sum?.value ?? 0))
-    ),
-    average: getPreviousMetric(
-      final.metrics.average,
-      round(
-        average(
-          final.series.map((item) => item.metrics.previous.average?.value ?? 0)
-        ),
-        2
-      )
-    ),
-    min: getPreviousMetric(
-      final.metrics.min,
-      min(final.series.map((item) => item.metrics.previous.min?.value ?? 0))
-    ),
-    max: getPreviousMetric(
-      final.metrics.max,
-      max(final.series.map((item) => item.metrics.previous.max?.value ?? 0))
-    ),
-  };
-
-  // Sort by sum
-  final.series = final.series.sort((a, b) => {
-    if (input.chartType === 'linear') {
-      const sumA = a.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
-      const sumB = b.data.reduce((acc, item) => acc + (item.count ?? 0), 0);
-      return sumB - sumA;
-    } else {
-      return b.metrics[input.metric] - a.metrics[input.metric];
-    }
-  });
-
-  return final;
-}
-
-export function getPreviousMetric(
-  current: number,
-  previous: number | null
-): PreviousValue {
-  if (previous === null) {
-    return null;
-  }
-
-  const diff = round(
-    ((current > previous
-      ? current / previous
-      : current < previous
-        ? previous / current
-        : 0) -
-      1) *
-      100,
-    1
-  );
-
-  return {
-    diff:
-      Number.isNaN(diff) || !Number.isFinite(diff) || current === previous
-        ? null
-        : diff,
-    state:
-      current > previous
-        ? 'positive'
-        : current < previous
-          ? 'negative'
-          : 'neutral',
-    value: previous,
-  };
-}

packages/validation/src/index.ts

@@ -71,6 +71,8 @@ export const zChartInput = z.object({
   projectId: z.string(),
   startDate: z.string().nullish(),
   endDate: z.string().nullish(),
+  limit: z.number().optional(),
+  offset: z.number().optional(),
 });
 
 export const zReportInput = zChartInput.extend({

packages/validation/src/types.validation.ts

@@ -31,9 +31,54 @@ export type IChartType = z.infer<typeof zChartType>;
 export type IChartMetric = z.infer<typeof zMetric>;
 export type IChartLineType = z.infer<typeof zLineType>;
 export type IChartRange = z.infer<typeof zRange>;
+export interface IChartInputWithDates extends IChartInput {
+  startDate: string;
+  endDate: string;
+}
 export type IGetChartDataInput = {
   event: IChartEvent;
   projectId: string;
   startDate: string;
   endDate: string;
 } & Omit<IChartInput, 'events' | 'name' | 'startDate' | 'endDate' | 'range'>;
+
+export type PreviousValue =
+  | {
+      value: number;
+      diff: number | null;
+      state: 'positive' | 'negative' | 'neutral';
+    }
+  | undefined;
+
+export type Metrics = {
+  sum: number;
+  average: number;
+  min: number;
+  max: number;
+  previous?: {
+    sum: PreviousValue;
+    average: PreviousValue;
+    min: PreviousValue;
+    max: PreviousValue;
+  };
+};
+
+export type IChartSerie = {
+  id: string;
+  name: string;
+  event: {
+    id: string;
+    name: string;
+  };
+  metrics: Metrics;
+  data: {
+    date: string;
+    count: number;
+    previous: PreviousValue;
+  }[];
+};
+
+export type FinalChart = {
+  series: IChartSerie[];
+  metrics: Metrics;
+};

pnpm-lock.yaml

@@ -799,6 +799,9 @@ importers:
 
   packages/common:
     dependencies:
+      '@openpanel/constants':
+        specifier: workspace:*
+        version: link:../constants
       date-fns:
         specifier: ^3.3.1
         version: 3.3.1