better handling dates in clickhouse

apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx

@@ -29,7 +29,7 @@ const SkipOnboarding = () => {
             text: 'Are you sure you want to skip onboarding? Since you do not have any projects, you will be logged out.',
             onConfirm() {
               auth.signOut();
-              router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL);
+              router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL!);
             },
           });
         }

apps/dashboard/src/app/providers.tsx

@@ -62,14 +62,14 @@ function AllProviders({ children }: { children: React.ReactNode }) {
       defaultTheme="light"
       disableTransitionOnChange
     >
-      {process.env.NEXT_PUBLIC_OP_CLIENT_ID && (
+      <OpenPanelComponent
-        <OpenPanelComponent
+        apiUrl="http://localhost:3333"
-          clientId={process.env.NEXT_PUBLIC_OP_CLIENT_ID}
+        clientId={'75479ffd-645b-43d2-a33a-2111a6f5ee00'}
-          trackScreenViews
+        trackScreenViews
-          trackOutgoingLinks
+        trackOutgoingLinks
-          trackAttributes
+        trackAttributes
-        />
+      />
-      )}
+
       <ReduxProvider store={storeRef.current}>
         <api.Provider client={trpcClient} queryClient={queryClient}>
           <QueryClientProvider client={queryClient}>

apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx

@@ -20,6 +20,8 @@ import type {
   IChartEventFilterValue,
 } from '@openpanel/validation';
 
+import { useOverviewOptions } from '../useOverviewOptions';
+
 export interface OverviewFiltersDrawerContentProps {
   projectId: string;
   nuqsOptions?: NuqsOptions;
@@ -33,10 +35,11 @@ export function OverviewFiltersDrawerContent({
   enableEventsFilter,
   mode,
 }: OverviewFiltersDrawerContentProps) {
+  const { interval, range } = useOverviewOptions();
   const [filters, setFilter] = useEventQueryFilters(nuqsOptions);
   const [event, setEvent] = useEventQueryNamesFilter(nuqsOptions);
-  const eventNames = useEventNames(projectId);
+  const eventNames = useEventNames({ projectId, interval, range });
-  const eventProperties = useEventProperties(projectId);
+  const eventProperties = useEventProperties({ projectId, interval, range });
   const profileProperties = useProfileProperties(projectId);
   const properties = mode === 'events' ? eventProperties : profileProperties;
 
@@ -113,11 +116,14 @@ export function FilterOptionEvent({
     operator: IChartEventFilterOperator
   ) => void;
 }) {
-  const values = useEventValues(
+  const { interval, range } = useOverviewOptions();
+  const values = useEventValues({
     projectId,
-    filter.name === 'path' ? 'screen_view' : 'session_start',
+    event: filter.name === 'path' ? 'screen_view' : 'session_start',
-    filter.name
+    property: filter.name,
-  );
+    interval,
+    range,
+  });
 
   return (
     <div className="flex items-center gap-2">

apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx

@@ -1,6 +1,6 @@
 import { Combobox } from '@/components/ui/combobox';
 import { useAppParams } from '@/hooks/useAppParams';
-import { useDispatch } from '@/redux';
+import { useDispatch, useSelector } from '@/redux';
 import { api } from '@/trpc/client';
 import { cn } from '@/utils/cn';
 import { DatabaseIcon } from 'lucide-react';
@@ -18,11 +18,14 @@ export function EventPropertiesCombobox({
 }: EventPropertiesComboboxProps) {
   const dispatch = useDispatch();
   const { projectId } = useAppParams();
-
+  const range = useSelector((state) => state.report.range);
+  const interval = useSelector((state) => state.report.interval);
   const query = api.chart.properties.useQuery(
     {
       event: event.name,
       projectId,
+      range,
+      interval,
     },
     {
       enabled: !!event.name,

apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx

@@ -16,9 +16,14 @@ import type { ReportEventMoreProps } from './ReportEventMore';
 export function ReportBreakdowns() {
   const { projectId } = useAppParams();
   const selectedBreakdowns = useSelector((state) => state.report.breakdowns);
+  const interval = useSelector((state) => state.report.interval);
+  const range = useSelector((state) => state.report.range);
+
   const dispatch = useDispatch();
   const propertiesQuery = api.chart.properties.useQuery({
     projectId,
+    range,
+    interval,
   });
   const propertiesCombobox = (propertiesQuery.data ?? []).map((item) => ({
     value: item,

apps/dashboard/src/components/report/sidebar/ReportEvents.tsx

@@ -29,13 +29,18 @@ import type { ReportEventMoreProps } from './ReportEventMore';
 export function ReportEvents() {
   const previous = useSelector((state) => state.report.previous);
   const selectedEvents = useSelector((state) => state.report.events);
-  const input = useSelector((state) => state.report);
+  const startDate = useSelector((state) => state.report.startDate);
+  const endDate = useSelector((state) => state.report.endDate);
+  const range = useSelector((state) => state.report.range);
+  const interval = useSelector((state) => state.report.interval);
   const dispatch = useDispatch();
   const { projectId } = useAppParams();
-  const eventNames = useEventNames(projectId, {
+  const eventNames = useEventNames({
-    startDate: input.startDate,
+    projectId,
-    endDate: input.endDate,
+    startDate,
-    range: input.range,
+    endDate,
+    range,
+    interval,
   });
 
   const dispatchChangeEvent = useDebounceFn((event: IChartEvent) => {

apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx

@@ -26,7 +26,9 @@ interface FilterProps {
 
 export function FilterItem({ filter, event }: FilterProps) {
   const { projectId } = useAppParams();
-  const { range, startDate, endDate } = useSelector((state) => state.report);
+  const { range, startDate, endDate, interval } = useSelector(
+    (state) => state.report
+  );
   const getLabel = useMappings();
   const dispatch = useDispatch();
   const potentialValues = api.chart.values.useQuery({
@@ -34,6 +36,7 @@ export function FilterItem({ filter, event }: FilterProps) {
     property: filter.name,
     projectId,
     range,
+    interval,
     startDate,
     endDate,
   });

apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx

@@ -15,7 +15,10 @@ interface FiltersComboboxProps {
 
 export function FiltersCombobox({ event }: FiltersComboboxProps) {
   const dispatch = useDispatch();
-  const { range, startDate, endDate } = useSelector((state) => state.report);
+  const interval = useSelector((state) => state.report.interval);
+  const range = useSelector((state) => state.report.range);
+  const startDate = useSelector((state) => state.report.startDate);
+  const endDate = useSelector((state) => state.report.endDate);
   const { projectId } = useAppParams();
 
   const query = api.chart.properties.useQuery(
@@ -23,6 +26,7 @@ export function FiltersCombobox({ event }: FiltersComboboxProps) {
       event: event.name,
       projectId,
       range,
+      interval,
       startDate,
       endDate,
     },

apps/dashboard/src/hooks/useEventNames.ts

@@ -1,10 +1,8 @@
 import { api } from '@/trpc/client';
 
-export function useEventNames(projectId: string, options?: any) {
+export function useEventNames(
-  const query = api.chart.events.useQuery({
+  params: Parameters<typeof api.chart.events.useQuery>[0]
-    projectId: projectId,
+) {
-    ...(options ? options : {}),
+  const query = api.chart.events.useQuery(params);
-  });
-
   return query.data ?? [];
 }

apps/dashboard/src/hooks/useEventProperties.ts

@@ -1,10 +1,9 @@
 import { api } from '@/trpc/client';
 
-export function useEventProperties(projectId: string, event?: string) {
+export function useEventProperties(
-  const query = api.chart.properties.useQuery({
+  params: Parameters<typeof api.chart.properties.useQuery>[0]
-    projectId: projectId,
+) {
-    event,
+  const query = api.chart.properties.useQuery(params);
-  });
 
   return query.data ?? [];
 }

apps/dashboard/src/hooks/useEventValues.ts

@@ -1,15 +1,8 @@
 import { api } from '@/trpc/client';
 
 export function useEventValues(
-  projectId: string,
+  params: Parameters<typeof api.chart.values.useQuery>[0]
-  event: string,
-  property: string
 ) {
-  const query = api.chart.values.useQuery({
+  const query = api.chart.values.useQuery(params);
-    projectId: projectId,
-    event,
-    property,
-  });
-
   return query.data?.values ?? [];
 }

packages/db/src/clickhouse-client.ts

@@ -1,5 +1,8 @@
 import type { ResponseJSON } from '@clickhouse/client';
 import { createClient } from '@clickhouse/client';
+import { escape } from 'sqlstring';
+
+import type { IInterval } from '@openpanel/validation';
 
 export const TABLE_NAMES = {
   events: 'events_v2',
@@ -126,6 +129,22 @@ export function formatClickhouseDate(
   return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
 }
 
+export function toDate(str: string, interval?: IInterval) {
+  if (!interval || interval === 'minute' || interval === 'hour') {
+    if (str.match(/\d{4}-\d{2}-\d{2}/)) {
+      return escape(str);
+    }
+
+    return str;
+  }
+
+  if (str.match(/\d{4}-\d{2}-\d{2}/)) {
+    return `toDate(${escape(str)})`;
+  }
+
+  return `toDate(${str})`;
+}
+
 export function convertClickhouseDateToJs(date: string) {
   return new Date(date.replace(' ', 'T') + 'Z');
 }

packages/db/src/services/chart.service.ts

@@ -6,7 +6,11 @@ import type {
   IGetChartDataInput,
 } from '@openpanel/validation';
 
-import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client';
+import {
+  formatClickhouseDate,
+  TABLE_NAMES,
+  toDate,
+} from '../clickhouse-client';
 import { createSqlBuilder } from '../sql-builder';
 
 function getPropertyKey(property: string) {
@@ -67,21 +71,11 @@ export function getChartSql({
   sb.groupBy.date = 'date';
 
   if (startDate) {
-    sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
+    sb.where.startDate = `${toDate('created_at', interval)} >= ${toDate(formatClickhouseDate(startDate), interval)}`;
-    // if (interval === 'minute' || interval === 'hour') {
-    //   sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
-    // } else {
-    //   sb.where.startDate = `toDate(created_at) >= '${formatClickhouseDate(startDate, true)}'`;
-    // }
   }
 
   if (endDate) {
-    sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
+    sb.where.endDate = `${toDate('created_at', interval)} <= ${toDate(formatClickhouseDate(endDate), interval)}`;
-    // if (interval === 'minute' || interval === 'hour') {
-    //   sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
-    // } else {
-    //   sb.where.endDate = `toDate(created_at) <= '${formatClickhouseDate(endDate, true)}'`;
-    // }
   }
 
   if (breakdowns.length > 0 && limit) {

packages/trpc/src/routers/chart.ts

@@ -1,22 +1,16 @@
-import { subMonths } from 'date-fns';
 import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
 import { escape } from 'sqlstring';
 import { z } from 'zod';
 
-import { average, max, min, round, slug, sum } from '@openpanel/common';
 import {
   chQuery,
   createSqlBuilder,
   db,
   formatClickhouseDate,
   TABLE_NAMES,
+  toDate,
 } from '@openpanel/db';
-import { zChartInput, zRange } from '@openpanel/validation';
+import { zChartInput, zRange, zTimeInterval } from '@openpanel/validation';
-import type {
-  FinalChart,
-  IChartInput,
-  PreviousValue,
-} from '@openpanel/validation';
 
 import { getProjectAccessCached } from '../access';
 import { TRPCAccessError } from '../errors';
@@ -34,7 +28,8 @@ export const chartRouter = createTRPCRouter({
     .input(
       z.object({
         projectId: z.string(),
-        range: zRange.default('30d'),
+        range: zRange,
+        interval: zTimeInterval,
         startDate: z.string().nullish(),
         endDate: z.string().nullish(),
       })
@@ -42,7 +37,7 @@ export const chartRouter = createTRPCRouter({
     .query(async ({ input: { projectId, ...input } }) => {
       const { startDate, endDate } = getChartStartEndDate(input);
       const events = await chQuery<{ name: string }>(
-        `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`
+        `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`
       );
 
       return [
@@ -58,7 +53,8 @@ export const chartRouter = createTRPCRouter({
       z.object({
         event: z.string().optional(),
         projectId: z.string(),
-        range: zRange.default('30d'),
+        range: zRange,
+        interval: zTimeInterval,
         startDate: z.string().nullish(),
         endDate: z.string().nullish(),
       })
@@ -69,7 +65,7 @@ export const chartRouter = createTRPCRouter({
         `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${
           event && event !== '*' ? `name = ${escape(event)} AND ` : ''
         } project_id = ${escape(projectId)} AND
-        created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`
+        ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`
       );
 
       const properties = events
@@ -111,7 +107,8 @@ export const chartRouter = createTRPCRouter({
         event: z.string(),
         property: z.string(),
         projectId: z.string(),
-        range: zRange.default('30d'),
+        range: zRange,
+        interval: zTimeInterval,
         startDate: z.string().nullish(),
         endDate: z.string().nullish(),
       })
@@ -137,7 +134,7 @@ export const chartRouter = createTRPCRouter({
         sb.select.values = `distinct ${property} as values`;
       }
 
-      sb.where.date = `created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
+      sb.where.date = `${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`;
 
       const events = await chQuery<{ values: string[] }>(getSql());