From 2be2ff3e12f2e0489b427627c1eb68064467ca1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= Date: Wed, 28 Aug 2024 10:53:23 +0200 Subject: [PATCH] better handling dates in clickhouse --- .../src/app/(onboarding)/skip-onboarding.tsx | 2 +- apps/dashboard/src/app/providers.tsx | 16 ++++++------ .../overview-filters-drawer-content.tsx | 18 ++++++++----- .../sidebar/EventPropertiesCombobox.tsx | 7 ++++-- .../report/sidebar/ReportBreakdowns.tsx | 5 ++++ .../report/sidebar/ReportEvents.tsx | 15 +++++++---- .../report/sidebar/filters/FilterItem.tsx | 5 +++- .../sidebar/filters/FiltersCombobox.tsx | 6 ++++- apps/dashboard/src/hooks/useEventNames.ts | 10 +++----- .../dashboard/src/hooks/useEventProperties.ts | 9 +++---- apps/dashboard/src/hooks/useEventValues.ts | 11 ++------ packages/db/src/clickhouse-client.ts | 19 ++++++++++++++ packages/db/src/services/chart.service.ts | 20 ++++++--------- packages/trpc/src/routers/chart.ts | 25 ++++++++----------- 14 files changed, 97 insertions(+), 71 deletions(-) diff --git a/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx b/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx index d5da4359..4a4f14ed 100644 --- a/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx +++ b/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx @@ -29,7 +29,7 @@ const SkipOnboarding = () => { text: 'Are you sure you want to skip onboarding? Since you do not have any projects, you will be logged out.', onConfirm() { auth.signOut(); - router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL); + router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL!); }, }); } diff --git a/apps/dashboard/src/app/providers.tsx b/apps/dashboard/src/app/providers.tsx index 918add31..ea7857dd 100644 --- a/apps/dashboard/src/app/providers.tsx +++ b/apps/dashboard/src/app/providers.tsx @@ -62,14 +62,14 @@ function AllProviders({ children }: { children: React.ReactNode }) { defaultTheme="light" disableTransitionOnChange > - {process.env.NEXT_PUBLIC_OP_CLIENT_ID && ( - - )} + + diff --git a/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx b/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx index 4da8cd88..0ac84a32 100644 --- a/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx +++ b/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx @@ -20,6 +20,8 @@ import type { IChartEventFilterValue, } from '@openpanel/validation'; +import { useOverviewOptions } from '../useOverviewOptions'; + export interface OverviewFiltersDrawerContentProps { projectId: string; nuqsOptions?: NuqsOptions; @@ -33,10 +35,11 @@ export function OverviewFiltersDrawerContent({ enableEventsFilter, mode, }: OverviewFiltersDrawerContentProps) { + const { interval, range } = useOverviewOptions(); const [filters, setFilter] = useEventQueryFilters(nuqsOptions); const [event, setEvent] = useEventQueryNamesFilter(nuqsOptions); - const eventNames = useEventNames(projectId); - const eventProperties = useEventProperties(projectId); + const eventNames = useEventNames({ projectId, interval, range }); + const eventProperties = useEventProperties({ projectId, interval, range }); const profileProperties = useProfileProperties(projectId); const properties = mode === 'events' ? eventProperties : profileProperties; @@ -113,11 +116,14 @@ export function FilterOptionEvent({ operator: IChartEventFilterOperator ) => void; }) { - const values = useEventValues( + const { interval, range } = useOverviewOptions(); + const values = useEventValues({ projectId, - filter.name === 'path' ? 'screen_view' : 'session_start', - filter.name - ); + event: filter.name === 'path' ? 'screen_view' : 'session_start', + property: filter.name, + interval, + range, + }); return (
diff --git a/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx b/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx index a021774a..8d8ecbd4 100644 --- a/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx +++ b/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx @@ -1,6 +1,6 @@ import { Combobox } from '@/components/ui/combobox'; import { useAppParams } from '@/hooks/useAppParams'; -import { useDispatch } from '@/redux'; +import { useDispatch, useSelector } from '@/redux'; import { api } from '@/trpc/client'; import { cn } from '@/utils/cn'; import { DatabaseIcon } from 'lucide-react'; @@ -18,11 +18,14 @@ export function EventPropertiesCombobox({ }: EventPropertiesComboboxProps) { const dispatch = useDispatch(); const { projectId } = useAppParams(); - + const range = useSelector((state) => state.report.range); + const interval = useSelector((state) => state.report.interval); const query = api.chart.properties.useQuery( { event: event.name, projectId, + range, + interval, }, { enabled: !!event.name, diff --git a/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx b/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx index 3be40b1e..0d1f5418 100644 --- a/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx +++ b/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx @@ -16,9 +16,14 @@ import type { ReportEventMoreProps } from './ReportEventMore'; export function ReportBreakdowns() { const { projectId } = useAppParams(); const selectedBreakdowns = useSelector((state) => state.report.breakdowns); + const interval = useSelector((state) => state.report.interval); + const range = useSelector((state) => state.report.range); + const dispatch = useDispatch(); const propertiesQuery = api.chart.properties.useQuery({ projectId, + range, + interval, }); const propertiesCombobox = (propertiesQuery.data ?? []).map((item) => ({ value: item, diff --git a/apps/dashboard/src/components/report/sidebar/ReportEvents.tsx b/apps/dashboard/src/components/report/sidebar/ReportEvents.tsx index ce1989e7..42c2e9a3 100644 --- a/apps/dashboard/src/components/report/sidebar/ReportEvents.tsx +++ b/apps/dashboard/src/components/report/sidebar/ReportEvents.tsx @@ -29,13 +29,18 @@ import type { ReportEventMoreProps } from './ReportEventMore'; export function ReportEvents() { const previous = useSelector((state) => state.report.previous); const selectedEvents = useSelector((state) => state.report.events); - const input = useSelector((state) => state.report); + const startDate = useSelector((state) => state.report.startDate); + const endDate = useSelector((state) => state.report.endDate); + const range = useSelector((state) => state.report.range); + const interval = useSelector((state) => state.report.interval); const dispatch = useDispatch(); const { projectId } = useAppParams(); - const eventNames = useEventNames(projectId, { - startDate: input.startDate, - endDate: input.endDate, - range: input.range, + const eventNames = useEventNames({ + projectId, + startDate, + endDate, + range, + interval, }); const dispatchChangeEvent = useDebounceFn((event: IChartEvent) => { diff --git a/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx b/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx index 5e228f80..bfa7de03 100644 --- a/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx +++ b/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx @@ -26,7 +26,9 @@ interface FilterProps { export function FilterItem({ filter, event }: FilterProps) { const { projectId } = useAppParams(); - const { range, startDate, endDate } = useSelector((state) => state.report); + const { range, startDate, endDate, interval } = useSelector( + (state) => state.report + ); const getLabel = useMappings(); const dispatch = useDispatch(); const potentialValues = api.chart.values.useQuery({ @@ -34,6 +36,7 @@ export function FilterItem({ filter, event }: FilterProps) { property: filter.name, projectId, range, + interval, startDate, endDate, }); diff --git a/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx b/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx index 0cdbc742..ba7c62c8 100644 --- a/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx +++ b/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx @@ -15,7 +15,10 @@ interface FiltersComboboxProps { export function FiltersCombobox({ event }: FiltersComboboxProps) { const dispatch = useDispatch(); - const { range, startDate, endDate } = useSelector((state) => state.report); + const interval = useSelector((state) => state.report.interval); + const range = useSelector((state) => state.report.range); + const startDate = useSelector((state) => state.report.startDate); + const endDate = useSelector((state) => state.report.endDate); const { projectId } = useAppParams(); const query = api.chart.properties.useQuery( @@ -23,6 +26,7 @@ export function FiltersCombobox({ event }: FiltersComboboxProps) { event: event.name, projectId, range, + interval, startDate, endDate, }, diff --git a/apps/dashboard/src/hooks/useEventNames.ts b/apps/dashboard/src/hooks/useEventNames.ts index 17d908d6..d224f886 100644 --- a/apps/dashboard/src/hooks/useEventNames.ts +++ b/apps/dashboard/src/hooks/useEventNames.ts @@ -1,10 +1,8 @@ import { api } from '@/trpc/client'; -export function useEventNames(projectId: string, options?: any) { - const query = api.chart.events.useQuery({ - projectId: projectId, - ...(options ? options : {}), - }); - +export function useEventNames( + params: Parameters[0] +) { + const query = api.chart.events.useQuery(params); return query.data ?? []; } diff --git a/apps/dashboard/src/hooks/useEventProperties.ts b/apps/dashboard/src/hooks/useEventProperties.ts index 732a1fca..bad5801b 100644 --- a/apps/dashboard/src/hooks/useEventProperties.ts +++ b/apps/dashboard/src/hooks/useEventProperties.ts @@ -1,10 +1,9 @@ import { api } from '@/trpc/client'; -export function useEventProperties(projectId: string, event?: string) { - const query = api.chart.properties.useQuery({ - projectId: projectId, - event, - }); +export function useEventProperties( + params: Parameters[0] +) { + const query = api.chart.properties.useQuery(params); return query.data ?? []; } diff --git a/apps/dashboard/src/hooks/useEventValues.ts b/apps/dashboard/src/hooks/useEventValues.ts index 6eda81be..e54dbf0c 100644 --- a/apps/dashboard/src/hooks/useEventValues.ts +++ b/apps/dashboard/src/hooks/useEventValues.ts @@ -1,15 +1,8 @@ import { api } from '@/trpc/client'; export function useEventValues( - projectId: string, - event: string, - property: string + params: Parameters[0] ) { - const query = api.chart.values.useQuery({ - projectId: projectId, - event, - property, - }); - + const query = api.chart.values.useQuery(params); return query.data?.values ?? []; } diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.ts index 9ea6128d..62968f92 100644 --- a/packages/db/src/clickhouse-client.ts +++ b/packages/db/src/clickhouse-client.ts @@ -1,5 +1,8 @@ import type { ResponseJSON } from '@clickhouse/client'; import { createClient } from '@clickhouse/client'; +import { escape } from 'sqlstring'; + +import type { IInterval } from '@openpanel/validation'; export const TABLE_NAMES = { events: 'events_v2', @@ -126,6 +129,22 @@ export function formatClickhouseDate( return date.toISOString().replace('T', ' ').replace(/Z+$/, ''); } +export function toDate(str: string, interval?: IInterval) { + if (!interval || interval === 'minute' || interval === 'hour') { + if (str.match(/\d{4}-\d{2}-\d{2}/)) { + return escape(str); + } + + return str; + } + + if (str.match(/\d{4}-\d{2}-\d{2}/)) { + return `toDate(${escape(str)})`; + } + + return `toDate(${str})`; +} + export function convertClickhouseDateToJs(date: string) { return new Date(date.replace(' ', 'T') + 'Z'); } diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts index 25394f59..3c6eb27b 100644 --- a/packages/db/src/services/chart.service.ts +++ b/packages/db/src/services/chart.service.ts @@ -6,7 +6,11 @@ import type { IGetChartDataInput, } from '@openpanel/validation'; -import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client'; +import { + formatClickhouseDate, + TABLE_NAMES, + toDate, +} from '../clickhouse-client'; import { createSqlBuilder } from '../sql-builder'; function getPropertyKey(property: string) { @@ -67,21 +71,11 @@ export function getChartSql({ sb.groupBy.date = 'date'; if (startDate) { - sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`; - // if (interval === 'minute' || interval === 'hour') { - // sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`; - // } else { - // sb.where.startDate = `toDate(created_at) >= '${formatClickhouseDate(startDate, true)}'`; - // } + sb.where.startDate = `${toDate('created_at', interval)} >= ${toDate(formatClickhouseDate(startDate), interval)}`; } if (endDate) { - sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`; - // if (interval === 'minute' || interval === 'hour') { - // sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`; - // } else { - // sb.where.endDate = `toDate(created_at) <= '${formatClickhouseDate(endDate, true)}'`; - // } + sb.where.endDate = `${toDate('created_at', interval)} <= ${toDate(formatClickhouseDate(endDate), interval)}`; } if (breakdowns.length > 0 && limit) { diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 4b3a3a9b..967074af 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -1,22 +1,16 @@ -import { subMonths } from 'date-fns'; import { flatten, map, pipe, prop, sort, uniq } from 'ramda'; import { escape } from 'sqlstring'; import { z } from 'zod'; -import { average, max, min, round, slug, sum } from '@openpanel/common'; import { chQuery, createSqlBuilder, db, formatClickhouseDate, TABLE_NAMES, + toDate, } from '@openpanel/db'; -import { zChartInput, zRange } from '@openpanel/validation'; -import type { - FinalChart, - IChartInput, - PreviousValue, -} from '@openpanel/validation'; +import { zChartInput, zRange, zTimeInterval } from '@openpanel/validation'; import { getProjectAccessCached } from '../access'; import { TRPCAccessError } from '../errors'; @@ -34,7 +28,8 @@ export const chartRouter = createTRPCRouter({ .input( z.object({ projectId: z.string(), - range: zRange.default('30d'), + range: zRange, + interval: zTimeInterval, startDate: z.string().nullish(), endDate: z.string().nullish(), }) @@ -42,7 +37,7 @@ export const chartRouter = createTRPCRouter({ .query(async ({ input: { projectId, ...input } }) => { const { startDate, endDate } = getChartStartEndDate(input); const events = await chQuery<{ name: string }>( - `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');` + `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};` ); return [ @@ -58,7 +53,8 @@ export const chartRouter = createTRPCRouter({ z.object({ event: z.string().optional(), projectId: z.string(), - range: zRange.default('30d'), + range: zRange, + interval: zTimeInterval, startDate: z.string().nullish(), endDate: z.string().nullish(), }) @@ -69,7 +65,7 @@ export const chartRouter = createTRPCRouter({ `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${ event && event !== '*' ? `name = ${escape(event)} AND ` : '' } project_id = ${escape(projectId)} AND - created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');` + ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};` ); const properties = events @@ -111,7 +107,8 @@ export const chartRouter = createTRPCRouter({ event: z.string(), property: z.string(), projectId: z.string(), - range: zRange.default('30d'), + range: zRange, + interval: zTimeInterval, startDate: z.string().nullish(), endDate: z.string().nullish(), }) @@ -137,7 +134,7 @@ export const chartRouter = createTRPCRouter({ sb.select.values = `distinct ${property} as values`; } - sb.where.date = `created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`; + sb.where.date = `${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`; const events = await chQuery<{ values: string[] }>(getSql());