zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

better handling dates in clickhouse

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-08-28 10:53:23 +02:00
parent df05e2dab3
commit 2be2ff3e12
14 changed files with 97 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx
View File

@@ -29,7 +29,7 @@ const SkipOnboarding = () => {
text: 'Are you sure you want to skip onboarding? Since you do not have any projects, you will be logged out.',
onConfirm() {
auth.signOut();
router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL);
router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL!);
},
});
}

16
apps/dashboard/src/app/providers.tsx
View File

@@ -62,14 +62,14 @@ function AllProviders({ children }: { children: React.ReactNode }) {
defaultTheme="light"
disableTransitionOnChange
>
{process.env.NEXT_PUBLIC_OP_CLIENT_ID && (
<OpenPanelComponent
clientId={process.env.NEXT_PUBLIC_OP_CLIENT_ID}
trackScreenViews
trackOutgoingLinks
trackAttributes
/>
)}
<OpenPanelComponent
apiUrl="http://localhost:3333"
clientId={'75479ffd-645b-43d2-a33a-2111a6f5ee00'}
trackScreenViews
trackOutgoingLinks
trackAttributes
/>
<ReduxProvider store={storeRef.current}>
<api.Provider client={trpcClient} queryClient={queryClient}>
<QueryClientProvider client={queryClient}>

18
apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx
View File

@@ -20,6 +20,8 @@ import type {
IChartEventFilterValue,
} from '@openpanel/validation';
import { useOverviewOptions } from '../useOverviewOptions';
export interface OverviewFiltersDrawerContentProps {
projectId: string;
nuqsOptions?: NuqsOptions;
@@ -33,10 +35,11 @@ export function OverviewFiltersDrawerContent({
enableEventsFilter,
mode,
}: OverviewFiltersDrawerContentProps) {
const { interval, range } = useOverviewOptions();
const [filters, setFilter] = useEventQueryFilters(nuqsOptions);
const [event, setEvent] = useEventQueryNamesFilter(nuqsOptions);
const eventNames = useEventNames(projectId);
const eventProperties = useEventProperties(projectId);
const eventNames = useEventNames({ projectId, interval, range });
const eventProperties = useEventProperties({ projectId, interval, range });
const profileProperties = useProfileProperties(projectId);
const properties = mode === 'events' ? eventProperties : profileProperties;
@@ -113,11 +116,14 @@ export function FilterOptionEvent({
operator: IChartEventFilterOperator
) => void;
}) {
const values = useEventValues(
const { interval, range } = useOverviewOptions();
const values = useEventValues({
projectId,
filter.name === 'path' ? 'screen_view' : 'session_start',
filter.name
);
event: filter.name === 'path' ? 'screen_view' : 'session_start',
property: filter.name,
interval,
range,
});
return (
<div className="flex items-center gap-2">

7
apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx
View File

@@ -1,6 +1,6 @@
import { Combobox } from '@/components/ui/combobox';
import { useAppParams } from '@/hooks/useAppParams';
import { useDispatch } from '@/redux';
import { useDispatch, useSelector } from '@/redux';
import { api } from '@/trpc/client';
import { cn } from '@/utils/cn';
import { DatabaseIcon } from 'lucide-react';
@@ -18,11 +18,14 @@ export function EventPropertiesCombobox({
}: EventPropertiesComboboxProps) {
const dispatch = useDispatch();
const { projectId } = useAppParams();
const range = useSelector((state) => state.report.range);
const interval = useSelector((state) => state.report.interval);
const query = api.chart.properties.useQuery(
{
event: event.name,
projectId,
range,
interval,
},
{
enabled: !!event.name,

5
apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx
View File

@@ -16,9 +16,14 @@ import type { ReportEventMoreProps } from './ReportEventMore';
export function ReportBreakdowns() {
const { projectId } = useAppParams();
const selectedBreakdowns = useSelector((state) => state.report.breakdowns);
const interval = useSelector((state) => state.report.interval);
const range = useSelector((state) => state.report.range);
const dispatch = useDispatch();
const propertiesQuery = api.chart.properties.useQuery({
projectId,
range,
interval,
});
const propertiesCombobox = (propertiesQuery.data ?? []).map((item) => ({
value: item,

15
apps/dashboard/src/components/report/sidebar/ReportEvents.tsx
View File

@@ -29,13 +29,18 @@ import type { ReportEventMoreProps } from './ReportEventMore';
export function ReportEvents() {
const previous = useSelector((state) => state.report.previous);
const selectedEvents = useSelector((state) => state.report.events);
const input = useSelector((state) => state.report);
const startDate = useSelector((state) => state.report.startDate);
const endDate = useSelector((state) => state.report.endDate);
const range = useSelector((state) => state.report.range);
const interval = useSelector((state) => state.report.interval);
const dispatch = useDispatch();
const { projectId } = useAppParams();
const eventNames = useEventNames(projectId, {
startDate: input.startDate,
endDate: input.endDate,
range: input.range,
const eventNames = useEventNames({
projectId,
startDate,
endDate,
range,
interval,
});
const dispatchChangeEvent = useDebounceFn((event: IChartEvent) => {

5
apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx
View File

@@ -26,7 +26,9 @@ interface FilterProps {
export function FilterItem({ filter, event }: FilterProps) {
const { projectId } = useAppParams();
const { range, startDate, endDate } = useSelector((state) => state.report);
const { range, startDate, endDate, interval } = useSelector(
(state) => state.report
);
const getLabel = useMappings();
const dispatch = useDispatch();
const potentialValues = api.chart.values.useQuery({
@@ -34,6 +36,7 @@ export function FilterItem({ filter, event }: FilterProps) {
property: filter.name,
projectId,
range,
interval,
startDate,
endDate,
});

6
apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx
View File

@@ -15,7 +15,10 @@ interface FiltersComboboxProps {
export function FiltersCombobox({ event }: FiltersComboboxProps) {
const dispatch = useDispatch();
const { range, startDate, endDate } = useSelector((state) => state.report);
const interval = useSelector((state) => state.report.interval);
const range = useSelector((state) => state.report.range);
const startDate = useSelector((state) => state.report.startDate);
const endDate = useSelector((state) => state.report.endDate);
const { projectId } = useAppParams();
const query = api.chart.properties.useQuery(
@@ -23,6 +26,7 @@ export function FiltersCombobox({ event }: FiltersComboboxProps) {
event: event.name,
projectId,
range,
interval,
startDate,
endDate,
},

10
apps/dashboard/src/hooks/useEventNames.ts
View File

@@ -1,10 +1,8 @@
import { api } from '@/trpc/client';
export function useEventNames(projectId: string, options?: any) {
const query = api.chart.events.useQuery({
projectId: projectId,
...(options ? options : {}),
});
export function useEventNames(
params: Parameters<typeof api.chart.events.useQuery>[0]
) {
const query = api.chart.events.useQuery(params);
return query.data ?? [];
}

9
apps/dashboard/src/hooks/useEventProperties.ts
View File

@@ -1,10 +1,9 @@
import { api } from '@/trpc/client';
export function useEventProperties(projectId: string, event?: string) {
const query = api.chart.properties.useQuery({
projectId: projectId,
event,
});
export function useEventProperties(
params: Parameters<typeof api.chart.properties.useQuery>[0]
) {
const query = api.chart.properties.useQuery(params);
return query.data ?? [];
}

11
apps/dashboard/src/hooks/useEventValues.ts
View File

@@ -1,15 +1,8 @@
import { api } from '@/trpc/client';
export function useEventValues(
projectId: string,
event: string,
property: string
params: Parameters<typeof api.chart.values.useQuery>[0]
) {
const query = api.chart.values.useQuery({
projectId: projectId,
event,
property,
});
const query = api.chart.values.useQuery(params);
return query.data?.values ?? [];
}

19
packages/db/src/clickhouse-client.ts
View File

@@ -1,5 +1,8 @@
import type { ResponseJSON } from '@clickhouse/client';
import { createClient } from '@clickhouse/client';
import { escape } from 'sqlstring';
import type { IInterval } from '@openpanel/validation';
export const TABLE_NAMES = {
events: 'events_v2',
@@ -126,6 +129,22 @@ export function formatClickhouseDate(
return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
}
export function toDate(str: string, interval?: IInterval) {
if (!interval || interval === 'minute' || interval === 'hour') {
if (str.match(/\d{4}-\d{2}-\d{2}/)) {
return escape(str);
}
return str;
}
if (str.match(/\d{4}-\d{2}-\d{2}/)) {
return `toDate(${escape(str)})`;
}
return `toDate(${str})`;
}
export function convertClickhouseDateToJs(date: string) {
return new Date(date.replace(' ', 'T') + 'Z');
}

20
packages/db/src/services/chart.service.ts
View File

@@ -6,7 +6,11 @@ import type {
IGetChartDataInput,
} from '@openpanel/validation';
import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client';
import {
formatClickhouseDate,
TABLE_NAMES,
toDate,
} from '../clickhouse-client';
import { createSqlBuilder } from '../sql-builder';
function getPropertyKey(property: string) {
@@ -67,21 +71,11 @@ export function getChartSql({
sb.groupBy.date = 'date';
if (startDate) {
sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
// if (interval === 'minute' || interval === 'hour') {
// sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
// } else {
// sb.where.startDate = `toDate(created_at) >= '${formatClickhouseDate(startDate, true)}'`;
// }
sb.where.startDate = `${toDate('created_at', interval)} >= ${toDate(formatClickhouseDate(startDate), interval)}`;
}
if (endDate) {
sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
// if (interval === 'minute' || interval === 'hour') {
// sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
// } else {
// sb.where.endDate = `toDate(created_at) <= '${formatClickhouseDate(endDate, true)}'`;
// }
sb.where.endDate = `${toDate('created_at', interval)} <= ${toDate(formatClickhouseDate(endDate), interval)}`;
}
if (breakdowns.length > 0 && limit) {

25
packages/trpc/src/routers/chart.ts
View File

@@ -1,22 +1,16 @@
import { subMonths } from 'date-fns';
import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
import { escape } from 'sqlstring';
import { z } from 'zod';
import { average, max, min, round, slug, sum } from '@openpanel/common';
import {
chQuery,
createSqlBuilder,
db,
formatClickhouseDate,
TABLE_NAMES,
toDate,
} from '@openpanel/db';
import { zChartInput, zRange } from '@openpanel/validation';
import type {
FinalChart,
IChartInput,
PreviousValue,
} from '@openpanel/validation';
import { zChartInput, zRange, zTimeInterval } from '@openpanel/validation';
import { getProjectAccessCached } from '../access';
import { TRPCAccessError } from '../errors';
@@ -34,7 +28,8 @@ export const chartRouter = createTRPCRouter({
.input(
z.object({
projectId: z.string(),
range: zRange.default('30d'),
range: zRange,
interval: zTimeInterval,
startDate: z.string().nullish(),
endDate: z.string().nullish(),
})
@@ -42,7 +37,7 @@ export const chartRouter = createTRPCRouter({
.query(async ({ input: { projectId, ...input } }) => {
const { startDate, endDate } = getChartStartEndDate(input);
const events = await chQuery<{ name: string }>(
`SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`
`SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`
);
return [
@@ -58,7 +53,8 @@ export const chartRouter = createTRPCRouter({
z.object({
event: z.string().optional(),
projectId: z.string(),
range: zRange.default('30d'),
range: zRange,
interval: zTimeInterval,
startDate: z.string().nullish(),
endDate: z.string().nullish(),
})
@@ -69,7 +65,7 @@ export const chartRouter = createTRPCRouter({
`SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${
event && event !== '*' ? `name = ${escape(event)} AND ` : ''
} project_id = ${escape(projectId)} AND
created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`
${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`
);
const properties = events
@@ -111,7 +107,8 @@ export const chartRouter = createTRPCRouter({
event: z.string(),
property: z.string(),
projectId: z.string(),
range: zRange.default('30d'),
range: zRange,
interval: zTimeInterval,
startDate: z.string().nullish(),
endDate: z.string().nullish(),
})
@@ -137,7 +134,7 @@ export const chartRouter = createTRPCRouter({
sb.select.values = `distinct ${property} as values`;
}
sb.where.date = `created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
sb.where.date = `${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`;
const events = await chQuery<{ values: string[] }>(getSql());