zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: default last/first seen broken when clickhouse defaults to 1970

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-02 09:34:23 +01:00
parent 647ac2a4af
commit 8c377c2066
6 changed files with 102 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/common/src/date.ts
View File

@@ -1,6 +1,6 @@
import { DateTime } from 'luxon';
export { DateTime } from 'luxon';
export { DateTime };
export type { DateTime };
export function getTime(date: string | number | Date) {
return new Date(date).getTime();

5
packages/db/src/clickhouse/client.ts
View File

@@ -290,3 +290,8 @@ export function toDate(str: string, interval?: IInterval) {
export function convertClickhouseDateToJs(date: string) {
return new Date(`${date.replace(' ', 'T')}Z`);
}
const ROLLUP_DATE_PREFIX = '1970-01-01';
export function isClickhouseDefaultMinDate(date: string): boolean {
return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');
}

78
packages/db/src/services/overview.service.ts
View File

@@ -1,14 +1,13 @@
import { average, sum } from '@openpanel/common';
import { chartColors } from '@openpanel/constants';
import { getCache } from '@openpanel/redis';
import { type IChartEventFilter, zTimeInterval } from '@openpanel/validation';
import { omit } from 'ramda';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import {
TABLE_NAMES,
ch,
convertClickhouseDateToJs,
isClickhouseDefaultMinDate,
TABLE_NAMES,
} from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
import {
@@ -172,24 +171,12 @@ export type IGetMapDataInput = z.infer<typeof zGetMapDataInput> & {
export class OverviewService {
constructor(private client: typeof ch) {}
// Helper methods
private isRollupRow(date: string): boolean {
// The rollup row has date 1970-01-01 00:00:00 (epoch) from ClickHouse.
// After transform with `new Date().toISOString()`, this becomes an ISO string.
// Due to timezone handling in JavaScript's Date constructor (which interprets
// the input as local time), the UTC date might become:
// - 1969-12-31T... for positive UTC offsets (e.g., UTC+8)
// - 1970-01-01T... for UTC or negative offsets
// We check for both year prefixes to handle all server timezones.
return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');
}
private getFillConfig(interval: string, startDate: string, endDate: string) {
const useDateOnly = ['month', 'week'].includes(interval);
return {
from: clix.toStartOf(
clix.datetime(startDate, useDateOnly ? 'toDate' : 'toDateTime'),
interval as any,
interval as any
),
to: clix.datetime(endDate, useDateOnly ? 'toDate' : 'toDateTime'),
step: clix.toInterval('1', interval as any),
@@ -234,12 +221,12 @@ export class OverviewService {
private mergeRevenueIntoSeries<T extends { date: string }>(
series: T[],
revenueData: { date: string; total_revenue: number }[],
revenueData: { date: string; total_revenue: number }[]
): (T & { total_revenue: number })[] {
const revenueByDate = new Map(
revenueData
.filter((r) => !this.isRollupRow(r.date))
.map((r) => [r.date, r.total_revenue]),
.filter((r) => !isClickhouseDefaultMinDate(r.date))
.map((r) => [r.date, r.total_revenue])
);
return series.map((row) => ({
...row,
@@ -248,10 +235,11 @@ export class OverviewService {
}
private getOverallRevenue(
revenueData: { date: string; total_revenue: number }[],
revenueData: { date: string; total_revenue: number }[]
): number {
return (
revenueData.find((r) => this.isRollupRow(r.date))?.total_revenue ?? 0
revenueData.find((r) => isClickhouseDefaultMinDate(r.date))
?.total_revenue ?? 0
);
}
@@ -263,7 +251,7 @@ export class OverviewService {
startDate: string;
endDate: string;
timezone: string;
},
}
): ReturnType<typeof clix> {
if (!this.isPageFilter(params.filters)) {
query.rawWhere(this.getRawWhereClause('sessions', params.filters));
@@ -276,7 +264,7 @@ export class OverviewService {
.where(
'id',
'IN',
clix.exp('(SELECT session_id FROM distinct_sessions)'),
clix.exp('(SELECT session_id FROM distinct_sessions)')
);
}
@@ -475,14 +463,14 @@ export class OverviewService {
clix(this.client, timezone)
.select(['bounce_rate'])
.from('session_agg')
.where('date', '=', rollupDate),
.where('date', '=', rollupDate)
)
.with(
'daily_session_stats',
clix(this.client, timezone)
.select(['date', 'bounce_rate'])
.from('session_agg')
.where('date', '!=', rollupDate),
.where('date', '!=', rollupDate)
)
.with('overall_unique_visitors', overallUniqueVisitorsQuery)
.select<{
@@ -512,7 +500,7 @@ export class OverviewService {
.from(`${TABLE_NAMES.events} AS e`)
.leftJoin(
'daily_session_stats AS dss',
`${clix.toStartOf('e.created_at', interval as any)} = dss.date`,
`${clix.toStartOf('e.created_at', interval as any)} = dss.date`
)
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
@@ -551,7 +539,7 @@ export class OverviewService {
(item) =>
item.overall_bounce_rate !== null ||
item.overall_total_sessions !== null ||
item.overall_unique_visitors !== null,
item.overall_unique_visitors !== null
);
return {
@@ -560,11 +548,11 @@ export class OverviewService {
unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0,
total_sessions: anyRowWithData?.overall_total_sessions ?? 0,
avg_session_duration: average(
mainRes.map((item) => item.avg_session_duration),
mainRes.map((item) => item.avg_session_duration)
),
total_screen_views: sum(mainRes.map((item) => item.total_screen_views)),
views_per_session: average(
mainRes.map((item) => item.views_per_session),
mainRes.map((item) => item.views_per_session)
),
total_revenue: overallRevenue,
},
@@ -591,7 +579,7 @@ export class OverviewService {
return item;
}
return item;
}),
})
);
return Object.values(where).join(' AND ');
@@ -879,7 +867,7 @@ export class OverviewService {
startDate,
endDate,
timezone,
},
}
);
const timeSeriesData = await mainTimeSeriesQuery.execute();
@@ -1066,7 +1054,7 @@ export class OverviewService {
.from('paths_deduped_cte')
.having('length(paths)', '>=', 2)
// ONLY sessions starting with top entry pages
.having('paths[1]', 'IN', topEntryPages),
.having('paths[1]', 'IN', topEntryPages)
)
.select<{
source: string;
@@ -1081,8 +1069,8 @@ export class OverviewService {
])
.from(
clix.exp(
'(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)',
),
'(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)'
)
)
.groupBy(['source', 'target', 'step'])
.orderBy('step', 'ASC')
@@ -1143,7 +1131,9 @@ export class OverviewService {
for (const t of fromSource) {
// Skip self-loops
if (t.source === t.target) continue;
if (t.source === t.target) {
continue;
}
const targetNodeId = getNodeId(t.target, step + 1);
@@ -1180,7 +1170,9 @@ export class OverviewService {
}
// Stop if no more nodes to process
if (activeNodes.size === 0) break;
if (activeNodes.size === 0) {
break;
}
}
// Step 5: Filter links by threshold (0.25% of total sessions)
@@ -1235,22 +1227,24 @@ export class OverviewService {
})
.sort((a, b) => {
// Sort by step first, then by value descending
if (a.step !== b.step) return a.step - b.step;
if (a.step !== b.step) {
return a.step - b.step;
}
return b.value - a.value;
});
// Sanity check: Ensure all link endpoints exist in nodes
const nodeIds = new Set(finalNodes.map((n) => n.id));
const invalidLinks = filteredLinks.filter(
(link) => !nodeIds.has(link.source) || !nodeIds.has(link.target),
(link) => !(nodeIds.has(link.source) && nodeIds.has(link.target))
);
if (invalidLinks.length > 0) {
console.warn(
`UserJourney: Found ${invalidLinks.length} links with missing nodes`,
`UserJourney: Found ${invalidLinks.length} links with missing nodes`
);
// Remove invalid links
const validLinks = filteredLinks.filter(
(link) => nodeIds.has(link.source) && nodeIds.has(link.target),
(link) => nodeIds.has(link.source) && nodeIds.has(link.target)
);
return {
nodes: finalNodes,
@@ -1260,7 +1254,9 @@ export class OverviewService {
// Sanity check: Ensure steps are monotonic (should always be true, but verify)
const stepsValid = finalNodes.every((node, idx, arr) => {
if (idx === 0) return true;
if (idx === 0) {
return true;
}
return node.step! >= arr[idx - 1]!.step!;
});
if (!stepsValid) {

38
packages/db/src/services/profile.service.ts
View File

@@ -1,23 +1,21 @@
import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { strip, toObject } from '@openpanel/common';
import { cacheable } from '@openpanel/redis';
import type { IChartEventFilter } from '@openpanel/validation';
import { uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { profileBuffer } from '../buffers';
import {
TABLE_NAMES,
ch,
chQuery,
convertClickhouseDateToJs,
formatClickhouseDate,
isClickhouseDefaultMinDate,
TABLE_NAMES,
} from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
export type IProfileMetrics = {
lastSeen: Date;
firstSeen: Date;
export interface IProfileMetrics {
lastSeen: Date | null;
firstSeen: Date | null;
screenViews: number;
sessions: number;
durationAvg: number;
@@ -29,7 +27,7 @@ export type IProfileMetrics = {
conversionEvents: number;
avgTimeBetweenSessions: number;
revenue: number;
};
}
export function getProfileMetrics(profileId: string, projectId: string) {
return chQuery<
Omit<IProfileMetrics, 'lastSeen' | 'firstSeen'> & {
@@ -100,8 +98,12 @@ export function getProfileMetrics(profileId: string, projectId: string) {
.then((data) => {
return {
...data,
lastSeen: convertClickhouseDateToJs(data.lastSeen),
firstSeen: convertClickhouseDateToJs(data.firstSeen),
lastSeen: isClickhouseDefaultMinDate(data.lastSeen)
? null
: convertClickhouseDateToJs(data.lastSeen),
firstSeen: isClickhouseDefaultMinDate(data.firstSeen)
? null
: convertClickhouseDateToJs(data.firstSeen),
};
});
}
@@ -127,7 +129,7 @@ export async function getProfileById(id: string, projectId: string) {
last_value(is_external) as is_external,
last_value(properties) as properties,
last_value(created_at) as created_at
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`
);
if (!profile) {
@@ -169,7 +171,7 @@ export async function getProfiles(ids: string[], projectId: string) {
project_id = ${sqlstring.escape(projectId)} AND
id IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')})
GROUP BY id, project_id
`,
`
);
return data.map(transformProfile);
@@ -221,7 +223,7 @@ export async function getProfileListCount({
return data[0]?.count ?? 0;
}
export type IServiceProfile = {
export interface IServiceProfile {
id: string;
email: string;
avatar: string;
@@ -245,7 +247,7 @@ export type IServiceProfile = {
model?: string;
referrer?: string;
};
};
}
export interface IClickhouseProfile {
id: string;
@@ -289,7 +291,7 @@ export function transformProfile({
};
}
export async function upsertProfile(
export function upsertProfile(
{
id,
firstName,
@@ -300,7 +302,7 @@ export async function upsertProfile(
projectId,
isExternal,
}: IServiceUpsertProfile,
isFromEvent = false,
isFromEvent = false
) {
const profile: IClickhouseProfile = {
id,