zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

better handling dates in clickhouse

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-08-28 10:53:23 +02:00
parent df05e2dab3
commit 2be2ff3e12
14 changed files with 97 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
packages/db/src/clickhouse-client.ts
View File

@@ -1,5 +1,8 @@
import type { ResponseJSON } from '@clickhouse/client';
import { createClient } from '@clickhouse/client';
import { escape } from 'sqlstring';
import type { IInterval } from '@openpanel/validation';
export const TABLE_NAMES = {
events: 'events_v2',
@@ -126,6 +129,22 @@ export function formatClickhouseDate(
return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
}
export function toDate(str: string, interval?: IInterval) {
if (!interval || interval === 'minute' || interval === 'hour') {
if (str.match(/\d{4}-\d{2}-\d{2}/)) {
return escape(str);
}
return str;
}
if (str.match(/\d{4}-\d{2}-\d{2}/)) {
return `toDate(${escape(str)})`;
}
return `toDate(${str})`;
}
export function convertClickhouseDateToJs(date: string) {
return new Date(date.replace(' ', 'T') + 'Z');
}

20
packages/db/src/services/chart.service.ts
View File

@@ -6,7 +6,11 @@ import type {
IGetChartDataInput,
} from '@openpanel/validation';
import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client';
import {
formatClickhouseDate,
TABLE_NAMES,
toDate,
} from '../clickhouse-client';
import { createSqlBuilder } from '../sql-builder';
function getPropertyKey(property: string) {
@@ -67,21 +71,11 @@ export function getChartSql({
sb.groupBy.date = 'date';
if (startDate) {
sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
// if (interval === 'minute' || interval === 'hour') {
// sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
// } else {
// sb.where.startDate = `toDate(created_at) >= '${formatClickhouseDate(startDate, true)}'`;
// }
sb.where.startDate = `${toDate('created_at', interval)} >= ${toDate(formatClickhouseDate(startDate), interval)}`;
}
if (endDate) {
sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
// if (interval === 'minute' || interval === 'hour') {
// sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
// } else {
// sb.where.endDate = `toDate(created_at) <= '${formatClickhouseDate(endDate, true)}'`;
// }
sb.where.endDate = `${toDate('created_at', interval)} <= ${toDate(formatClickhouseDate(endDate), interval)}`;
}
if (breakdowns.length > 0 && limit) {

25
packages/trpc/src/routers/chart.ts
View File

@@ -1,22 +1,16 @@
import { subMonths } from 'date-fns';
import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
import { escape } from 'sqlstring';
import { z } from 'zod';
import { average, max, min, round, slug, sum } from '@openpanel/common';
import {
chQuery,
createSqlBuilder,
db,
formatClickhouseDate,
TABLE_NAMES,
toDate,
} from '@openpanel/db';
import { zChartInput, zRange } from '@openpanel/validation';
import type {
FinalChart,
IChartInput,
PreviousValue,
} from '@openpanel/validation';
import { zChartInput, zRange, zTimeInterval } from '@openpanel/validation';
import { getProjectAccessCached } from '../access';
import { TRPCAccessError } from '../errors';
@@ -34,7 +28,8 @@ export const chartRouter = createTRPCRouter({
.input(
z.object({
projectId: z.string(),
range: zRange.default('30d'),
range: zRange,
interval: zTimeInterval,
startDate: z.string().nullish(),
endDate: z.string().nullish(),
})
@@ -42,7 +37,7 @@ export const chartRouter = createTRPCRouter({
.query(async ({ input: { projectId, ...input } }) => {
const { startDate, endDate } = getChartStartEndDate(input);
const events = await chQuery<{ name: string }>(
`SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`
`SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`
);
return [
@@ -58,7 +53,8 @@ export const chartRouter = createTRPCRouter({
z.object({
event: z.string().optional(),
projectId: z.string(),
range: zRange.default('30d'),
range: zRange,
interval: zTimeInterval,
startDate: z.string().nullish(),
endDate: z.string().nullish(),
})
@@ -69,7 +65,7 @@ export const chartRouter = createTRPCRouter({
`SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${
event && event !== '*' ? `name = ${escape(event)} AND ` : ''
} project_id = ${escape(projectId)} AND
created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`
${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`
);
const properties = events
@@ -111,7 +107,8 @@ export const chartRouter = createTRPCRouter({
event: z.string(),
property: z.string(),
projectId: z.string(),
range: zRange.default('30d'),
range: zRange,
interval: zTimeInterval,
startDate: z.string().nullish(),
endDate: z.string().nullish(),
})
@@ -137,7 +134,7 @@ export const chartRouter = createTRPCRouter({
sb.select.values = `distinct ${property} as values`;
}
sb.where.date = `created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
sb.where.date = `${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`;
const events = await chQuery<{ values: string[] }>(getSql());