feat(api): add insights endpoints

packages/constants/index.ts

@@ -1,4 +1,4 @@
-import { isSameDay, isSameMonth } from 'date-fns';
+import { differenceInDays, isSameDay, isSameMonth } from 'date-fns';
 
 export const DEFAULT_ASPECT_RATIO = 0.5625;
 export const NOT_SET_VALUE = '(not set)';
@@ -232,6 +232,9 @@ export function getDefaultIntervalByDates(
     if (isSameMonth(startDate, endDate)) {
       return 'day';
     }
+    if (differenceInDays(endDate, startDate) <= 31) {
+      return 'day';
+    }
     return 'month';
   }
 

packages/db/src/services/chart.service.ts

@@ -1,8 +1,10 @@
 import { escape } from 'sqlstring';
 
-import { stripLeadingAndTrailingSlashes } from '@openpanel/common';
+import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common';
 import type {
   IChartEventFilter,
+  IChartInput,
+  IChartRange,
   IGetChartDataInput,
 } from '@openpanel/validation';
 
@@ -441,3 +443,240 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
 
   return where;
 }
+
+export function getChartStartEndDate(
+  {
+    startDate,
+    endDate,
+    range,
+  }: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>,
+  timezone: string,
+) {
+  const ranges = getDatesFromRange(range, timezone);
+
+  if (startDate && endDate) {
+    return { startDate: startDate, endDate: endDate };
+  }
+
+  if (!startDate && endDate) {
+    return { startDate: ranges.startDate, endDate: endDate };
+  }
+
+  return ranges;
+}
+
+export function getDatesFromRange(range: IChartRange, timezone: string) {
+  if (range === '30min' || range === 'lastHour') {
+    const minutes = range === '30min' ? 30 : 60;
+    const startDate = DateTime.now()
+      .minus({ minute: minutes })
+      .startOf('minute')
+      .setZone(timezone)
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .setZone(timezone)
+      .endOf('minute')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === 'today') {
+    const startDate = DateTime.now()
+      .setZone(timezone)
+      .startOf('day')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .setZone(timezone)
+      .endOf('day')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === 'yesterday') {
+    const startDate = DateTime.now()
+      .minus({ day: 1 })
+      .setZone(timezone)
+      .startOf('day')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .minus({ day: 1 })
+      .setZone(timezone)
+      .endOf('day')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === '7d') {
+    const startDate = DateTime.now()
+      .minus({ day: 7 })
+      .setZone(timezone)
+      .startOf('day')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .setZone(timezone)
+      .endOf('day')
+      .plus({ millisecond: 1 })
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === '6m') {
+    const startDate = DateTime.now()
+      .minus({ month: 6 })
+      .setZone(timezone)
+      .startOf('day')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .setZone(timezone)
+      .endOf('day')
+      .plus({ millisecond: 1 })
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === '12m') {
+    const startDate = DateTime.now()
+      .minus({ month: 12 })
+      .setZone(timezone)
+      .startOf('month')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .setZone(timezone)
+      .endOf('month')
+      .plus({ millisecond: 1 })
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === 'monthToDate') {
+    const startDate = DateTime.now()
+      .setZone(timezone)
+      .startOf('month')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .setZone(timezone)
+      .endOf('day')
+      .plus({ millisecond: 1 })
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === 'lastMonth') {
+    const month = DateTime.now()
+      .minus({ month: 1 })
+      .setZone(timezone)
+      .startOf('month');
+
+    const startDate = month.toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = month
+      .endOf('month')
+      .plus({ millisecond: 1 })
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === 'yearToDate') {
+    const startDate = DateTime.now()
+      .setZone(timezone)
+      .startOf('year')
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = DateTime.now()
+      .setZone(timezone)
+      .endOf('day')
+      .plus({ millisecond: 1 })
+      .toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  if (range === 'lastYear') {
+    const year = DateTime.now().minus({ year: 1 }).setZone(timezone);
+    const startDate = year.startOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
+    const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
+
+    return {
+      startDate: startDate,
+      endDate: endDate,
+    };
+  }
+
+  // range === '30d'
+  const startDate = DateTime.now()
+    .minus({ day: 30 })
+    .setZone(timezone)
+    .startOf('day')
+    .toFormat('yyyy-MM-dd HH:mm:ss');
+  const endDate = DateTime.now()
+    .setZone(timezone)
+    .endOf('day')
+    .plus({ millisecond: 1 })
+    .toFormat('yyyy-MM-dd HH:mm:ss');
+
+  return {
+    startDate: startDate,
+    endDate: endDate,
+  };
+}
+
+export function getChartPrevStartEndDate({
+  startDate,
+  endDate,
+}: {
+  startDate: string;
+  endDate: string;
+}) {
+  let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff(
+    DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'),
+  );
+
+  // this will make sure our start and end date's are correct
+  // otherwise if a day ends with 23:59:59.999 and starts with 00:00:00.000
+  // the diff will be 23:59:59.999 and that will make the start date wrong
+  // so we add 1 millisecond to the diff
+  if ((diff.milliseconds / 1000) % 2 !== 0) {
+    diff = diff.plus({ millisecond: 1 });
+  }
+
+  return {
+    startDate: DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss')
+      .minus({ millisecond: diff.milliseconds })
+      .toFormat('yyyy-MM-dd HH:mm:ss'),
+    endDate: DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss')
+      .minus({ millisecond: diff.milliseconds })
+      .toFormat('yyyy-MM-dd HH:mm:ss'),
+  };
+}

packages/db/src/services/overview.service.ts

@@ -24,7 +24,6 @@ export const zGetTopPagesInput = z.object({
   filters: z.array(z.any()),
   startDate: z.string(),
   endDate: z.string(),
-  interval: zTimeInterval,
   cursor: z.number().optional(),
   limit: z.number().optional(),
 });
@@ -38,7 +37,6 @@ export const zGetTopEntryExitInput = z.object({
   filters: z.array(z.any()),
   startDate: z.string(),
   endDate: z.string(),
-  interval: zTimeInterval,
   mode: z.enum(['entry', 'exit']),
   cursor: z.number().optional(),
   limit: z.number().optional(),
@@ -53,7 +51,6 @@ export const zGetTopGenericInput = z.object({
   filters: z.array(z.any()),
   startDate: z.string(),
   endDate: z.string(),
-  interval: zTimeInterval,
   column: z.enum([
     // Referrers
     'referrer',
@@ -168,6 +165,16 @@ export class OverviewService {
       views_per_session: number;
     }[];
   }> {
+    console.log('-----------------');
+    console.log('getMetrics', {
+      projectId,
+      filters,
+      startDate,
+      endDate,
+      interval,
+      timezone,
+    });
+
     const where = this.getRawWhereClause('sessions', filters);
     if (this.isPageFilter(filters)) {
       // Session aggregation with bounce rates

packages/trpc/src/routers/chart.helpers.ts

@@ -20,7 +20,9 @@ import {
   chQuery,
   createSqlBuilder,
   formatClickhouseDate,
+  getChartPrevStartEndDate,
   getChartSql,
+  getChartStartEndDate,
   getEventFiltersWhereClause,
   getOrganizationSubscriptionChartEndDate,
   getSettingsForProject,
@@ -34,10 +36,6 @@ import type {
   IGetChartDataInput,
 } from '@openpanel/validation';
 
-function getEventLegend(event: IChartEvent) {
-  return event.displayName || event.name;
-}
-
 export function withFormula(
   { formula, events }: IChartInput,
   series: Awaited<ReturnType<typeof getChartSerie>>,
@@ -116,193 +114,6 @@ export function withFormula(
   ];
 }
 
-export function getDatesFromRange(range: IChartRange, timezone: string) {
-  if (range === '30min' || range === 'lastHour') {
-    const minutes = range === '30min' ? 30 : 60;
-    const startDate = DateTime.now()
-      .minus({ minute: minutes })
-      .startOf('minute')
-      .setZone(timezone)
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .setZone(timezone)
-      .endOf('minute')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === 'today') {
-    const startDate = DateTime.now()
-      .setZone(timezone)
-      .startOf('day')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .setZone(timezone)
-      .endOf('day')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === 'yesterday') {
-    const startDate = DateTime.now()
-      .minus({ day: 1 })
-      .setZone(timezone)
-      .startOf('day')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .minus({ day: 1 })
-      .setZone(timezone)
-      .endOf('day')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === '7d') {
-    const startDate = DateTime.now()
-      .minus({ day: 7 })
-      .setZone(timezone)
-      .startOf('day')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .setZone(timezone)
-      .endOf('day')
-      .plus({ millisecond: 1 })
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === '6m') {
-    const startDate = DateTime.now()
-      .minus({ month: 6 })
-      .setZone(timezone)
-      .startOf('day')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .setZone(timezone)
-      .endOf('day')
-      .plus({ millisecond: 1 })
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === '12m') {
-    const startDate = DateTime.now()
-      .minus({ month: 12 })
-      .setZone(timezone)
-      .startOf('month')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .setZone(timezone)
-      .endOf('month')
-      .plus({ millisecond: 1 })
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === 'monthToDate') {
-    const startDate = DateTime.now()
-      .setZone(timezone)
-      .startOf('month')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .setZone(timezone)
-      .endOf('day')
-      .plus({ millisecond: 1 })
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === 'lastMonth') {
-    const month = DateTime.now()
-      .minus({ month: 1 })
-      .setZone(timezone)
-      .startOf('month');
-
-    const startDate = month.toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = month
-      .endOf('month')
-      .plus({ millisecond: 1 })
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === 'yearToDate') {
-    const startDate = DateTime.now()
-      .setZone(timezone)
-      .startOf('year')
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = DateTime.now()
-      .setZone(timezone)
-      .endOf('day')
-      .plus({ millisecond: 1 })
-      .toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  if (range === 'lastYear') {
-    const year = DateTime.now().minus({ year: 1 }).setZone(timezone);
-    const startDate = year.startOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
-    const endDate = year.endOf('year').toFormat('yyyy-MM-dd HH:mm:ss');
-
-    return {
-      startDate: startDate,
-      endDate: endDate,
-    };
-  }
-
-  // range === '30d'
-  const startDate = DateTime.now()
-    .minus({ day: 30 })
-    .setZone(timezone)
-    .startOf('day')
-    .toFormat('yyyy-MM-dd HH:mm:ss');
-  const endDate = DateTime.now()
-    .setZone(timezone)
-    .endOf('day')
-    .plus({ millisecond: 1 })
-    .toFormat('yyyy-MM-dd HH:mm:ss');
-
-  return {
-    startDate: startDate,
-    endDate: endDate,
-  };
-}
-
 function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
   const filled = Array.from({ length: steps }, (_, index) => {
     const level = index + 1;
@@ -325,56 +136,6 @@ function fillFunnel(funnel: { level: number; count: number }[], steps: number) {
   return filled.reverse();
 }
 
-export function getChartStartEndDate(
-  {
-    startDate,
-    endDate,
-    range,
-  }: Pick<IChartInput, 'endDate' | 'startDate' | 'range'>,
-  timezone: string,
-) {
-  const ranges = getDatesFromRange(range, timezone);
-
-  if (startDate && endDate) {
-    return { startDate: startDate, endDate: endDate };
-  }
-
-  if (!startDate && endDate) {
-    return { startDate: ranges.startDate, endDate: endDate };
-  }
-
-  return ranges;
-}
-
-export function getChartPrevStartEndDate({
-  startDate,
-  endDate,
-}: {
-  startDate: string;
-  endDate: string;
-}) {
-  let diff = DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss').diff(
-    DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss'),
-  );
-
-  // this will make sure our start and end date's are correct
-  // otherwise if a day ends with 23:59:59.999 and starts with 00:00:00.000
-  // the diff will be 23:59:59.999 and that will make the start date wrong
-  // so we add 1 millisecond to the diff
-  if ((diff.milliseconds / 1000) % 2 !== 0) {
-    diff = diff.plus({ millisecond: 1 });
-  }
-
-  return {
-    startDate: DateTime.fromFormat(startDate, 'yyyy-MM-dd HH:mm:ss')
-      .minus({ millisecond: diff.milliseconds })
-      .toFormat('yyyy-MM-dd HH:mm:ss'),
-    endDate: DateTime.fromFormat(endDate, 'yyyy-MM-dd HH:mm:ss')
-      .minus({ millisecond: diff.milliseconds })
-      .toFormat('yyyy-MM-dd HH:mm:ss'),
-  };
-}
-
 export async function getFunnelData({
   projectId,
   startDate,

packages/trpc/src/routers/chart.ts

@@ -10,13 +10,13 @@ import {
   chQuery,
   clix,
   conversionService,
-  createSqlBuilder,
   db,
   funnelService,
+  getChartPrevStartEndDate,
+  getChartStartEndDate,
   getEventMetasCached,
   getSelectPropertyKey,
   getSettingsForProject,
-  toDate,
 } from '@openpanel/db';
 import {
   zChartInput,
@@ -40,11 +40,7 @@ import {
   protectedProcedure,
   publicProcedure,
 } from '../trpc';
-import {
-  getChart,
-  getChartPrevStartEndDate,
-  getChartStartEndDate,
-} from './chart.helpers';
+import { getChart } from './chart.helpers';
 
 function utc(date: string | Date) {
   if (typeof date === 'string') {

packages/trpc/src/routers/event.ts

@@ -10,6 +10,7 @@ import {
   db,
   eventService,
   formatClickhouseDate,
+  getChartStartEndDate,
   getConversionEventNames,
   getEventList,
   getEventMetasCached,
@@ -28,7 +29,6 @@ import { clone } from 'ramda';
 import { getProjectAccessCached } from '../access';
 import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
-import { getChartStartEndDate } from './chart.helpers';
 
 export const eventRouter = createTRPCRouter({
   updateEventMeta: protectedProcedure
@@ -289,7 +289,6 @@ export const eventRouter = createTRPCRouter({
         filters: input.filters,
         startDate,
         endDate,
-        interval: input.interval,
         cursor: input.cursor || 1,
         limit: input.take,
         timezone,

packages/trpc/src/routers/overview.ts

@@ -1,4 +1,6 @@
 import {
+  getChartPrevStartEndDate,
+  getChartStartEndDate,
   getOrganizationSubscriptionChartEndDate,
   getSettingsForProject,
   overviewService,
@@ -10,10 +12,6 @@ import { type IChartRange, zRange } from '@openpanel/validation';
 import { format } from 'date-fns';
 import { z } from 'zod';
 import { cacheMiddleware, createTRPCRouter, publicProcedure } from '../trpc';
-import {
-  getChartPrevStartEndDate,
-  getChartStartEndDate,
-} from './chart.helpers';
 
 const cacher = cacheMiddleware((input) => {
   const range = input.range as IChartRange;

packages/trpc/src/routers/reference.ts

@@ -1,12 +1,16 @@
 import { z } from 'zod';
 
-import { db, getReferences, getSettingsForProject } from '@openpanel/db';
+import {
+  db,
+  getChartStartEndDate,
+  getReferences,
+  getSettingsForProject,
+} from '@openpanel/db';
 import { zCreateReference, zRange } from '@openpanel/validation';
 
 import { getProjectAccess } from '../access';
 import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
-import { getChartStartEndDate } from './chart.helpers';
 
 export const referenceRouter = createTRPCRouter({
   create: protectedProcedure