zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix timezones

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-05-03 09:46:13 +02:00
parent 33e92a0da0
commit e4e6cc3316
7 changed files with 148 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/common/index.ts
View File

@@ -6,3 +6,4 @@ export * from './src/names';
export * from './src/string';
export * from './src/math';
export * from './src/slug';
export * from './src/fill-series';

2
packages/common/package.json
View File

@@ -8,6 +8,7 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"date-fns": "^3.3.1",
"mathjs": "^12.3.2",
"ramda": "^0.29.1",
"slugify": "^1.6.6",
@@ -15,6 +16,7 @@
"unique-names-generator": "^4.7.1"
},
"devDependencies": {
"@openpanel/validation": "workspace:*",
"@openpanel/eslint-config": "workspace:*",
"@openpanel/prettier-config": "workspace:*",
"@openpanel/tsconfig": "workspace:*",

93
packages/common/src/fill-series.ts Normal file
View File

@@ -0,0 +1,93 @@
import {
addDays,
addHours,
addMinutes,
addMonths,
format,
parseISO,
startOfDay,
startOfHour,
startOfMinute,
startOfMonth,
} from 'date-fns';
import type { IInterval } from '@openpanel/validation';
// Define the data structure
interface DataEntry {
label: string;
count: number | null;
date: string;
}
// Function to round down the date to the nearest interval
function roundDate(date: Date, interval: IInterval): Date {
switch (interval) {
case 'minute':
return startOfMinute(date);
case 'hour':
return startOfHour(date);
case 'day':
return startOfDay(date);
case 'month':
return startOfMonth(date);
default:
return startOfMinute(date);
}
}
// Function to complete the timeline for each label
export function completeTimeline(
data: DataEntry[],
_startDate: string,
_endDate: string,
interval: IInterval
) {
const startDate = parseISO(_startDate);
const endDate = parseISO(_endDate);
// Group data by label
const labelsMap = new Map<string, Map<string, number>>();
data.forEach((entry) => {
const roundedDate = roundDate(parseISO(entry.date), interval);
const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss');
if (!labelsMap.has(entry.label)) {
labelsMap.set(entry.label, new Map());
}
const labelData = labelsMap.get(entry.label);
labelData?.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0));
});
// Complete the timeline for each label
const result: Record<string, DataEntry[]> = {};
labelsMap.forEach((counts, label) => {
let currentDate = roundDate(startDate, interval);
result[label] = [];
while (currentDate <= endDate) {
const dateKey = format(currentDate, 'yyyy-MM-dd HH:mm:ss');
result[label]!.push({
label: label,
date: dateKey,
count: counts.get(dateKey) || 0,
});
// Increment the current date based on the interval
switch (interval) {
case 'minute':
currentDate = addMinutes(currentDate, 1);
break;
case 'hour':
currentDate = addHours(currentDate, 1);
break;
case 'day':
currentDate = addDays(currentDate, 1);
break;
case 'month':
currentDate = addMonths(currentDate, 1);
break;
}
}
});
return result;
}

12
packages/db/src/services/chart.service.ts
View File

@@ -23,35 +23,29 @@ export function getChartSql({
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.projectId = `project_id = ${escape(projectId)}`;
let labelValue = escape('*');
if (event.name !== '*') {
labelValue = `${escape(event.name)}`;
sb.select.label = `${labelValue} as label`;
sb.where.eventName = `name = ${labelValue}`;
sb.select.label = `${escape(event.name)} as label`;
sb.where.eventName = `name = ${escape(event.name)}`;
} else {
sb.select.label = `${labelValue} as label`;
sb.select.label = `'*' as label`;
}
sb.select.count = `count(*) as count`;
switch (interval) {
case 'minute': {
sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'hour': {
sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'day': {
sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'month': {
sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
}

113
packages/trpc/src/routers/chart.helpers.ts
View File

@@ -17,7 +17,7 @@ import * as mathjs from 'mathjs';
import { repeat, reverse, sort } from 'ramda';
import { escape } from 'sqlstring';
import { round } from '@openpanel/common';
import { completeTimeline, round } from '@openpanel/common';
import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
import {
chQuery,
@@ -133,88 +133,59 @@ const toDynamicISODateWithTZ = (
// It will be converted to the correct timezone on the client
return date.replace(' ', 'T');
}
return `${date}T00:00:00`;
return `${date}T00:00:00Z`;
};
export async function getChartData(payload: IGetChartDataInput) {
let result = await chQuery<ResultItem>(getChartSql(payload));
if (result.length === 0 && payload.breakdowns.length > 0) {
result = await chQuery<ResultItem>(
getChartSql({
...payload,
breakdowns: [],
})
);
async function getSeries() {
const result = await chQuery<ResultItem>(getChartSql(payload));
if (result.length === 0 && payload.breakdowns.length > 0) {
return await chQuery<ResultItem>(
getChartSql({
...payload,
breakdowns: [],
})
);
}
return result;
}
// group by sql label
const series = result.reduce(
(acc, item) => {
// If we fill empty spots in the timeline (clickhouse) we wont get a label back
// take the event name as label
if (!item.label && item.count === 0) {
item.label = payload.event.name;
}
return getSeries()
.then((data) =>
completeTimeline(
data.map((item) => {
const label = item.label?.trim() || NOT_SET_VALUE;
const label = item.label?.trim() || NOT_SET_VALUE;
// item.label can be null when using breakdowns on a property
// that doesn't exist on all events
if (label) {
if (acc[label]) {
acc[label]?.push(item);
} else {
acc[label] = [item];
}
}
return {
...acc,
};
},
{} as Record<string, ResultItem[]>
);
return Object.keys(series).map((key) => {
// If we have breakdowns, we want to use the breakdown key as the legend
// But only if it successfully broke it down, otherwise we use the getEventLabel
const isBreakdown =
payload.breakdowns.length && !alphabetIds.includes(key as 'A');
const serieName = isBreakdown ? key : getEventLegend(payload.event);
const data =
payload.chartType === 'area' ||
payload.chartType === 'linear' ||
payload.chartType === 'histogram' ||
payload.chartType === 'metric' ||
payload.chartType === 'pie' ||
payload.chartType === 'bar'
? (series[key] ?? []).map((item) => {
return {
label: serieName,
count: item.count ? round(item.count) : null,
date: toDynamicISODateWithTZ(
item.date,
payload.startDate,
payload.interval
),
};
})
: (series[key] ?? []).map((item) => ({
label: item.label,
return {
...item,
count: item.count ? round(item.count) : null,
label,
};
}),
payload.startDate,
payload.endDate,
payload.interval
)
)
.then((series) => {
return Object.keys(series).map((label) => {
const isBreakdown =
payload.breakdowns.length && !alphabetIds.includes(label as 'A');
const serieLabel = isBreakdown ? label : getEventLegend(payload.event);
return {
name: serieLabel,
event: payload.event,
data: series[label]!.map((item) => ({
...item,
date: toDynamicISODateWithTZ(
item.date,
payload.startDate,
payload.interval
),
}));
return {
name: serieName,
event: payload.event,
data,
};
});
})),
};
});
});
}
export function getDatesFromRange(range: IChartRange) {

2
packages/trpc/src/routers/chart.ts
View File

@@ -184,7 +184,7 @@ export const chartRouter = createTRPCRouter({
}),
// TODO: Make this private
chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => {
chart: publicProcedure.input(zChartInput).query(async ({ input }) => {
const currentPeriod = getChartStartEndDate(input);
const previousPeriod = getChartPrevStartEndDate({
range: input.range,

6
pnpm-lock.yaml generated
View File

@@ -763,6 +763,9 @@ importers:
packages/common:
dependencies:
date-fns:
specifier: ^3.3.1
version: 3.3.1
mathjs:
specifier: ^12.3.2
version: 12.3.2
@@ -788,6 +791,9 @@ importers:
'@openpanel/tsconfig':
specifier: workspace:*
version: link:../../tooling/typescript
'@openpanel/validation':
specifier: workspace:*
version: link:../validation
'@types/node':
specifier: ^18.16.0
version: 18.19.17