From 8c377c2066a328a9ddd110e978bc407b129db139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= Date: Mon, 2 Mar 2026 09:34:23 +0100 Subject: [PATCH] fix: default last/first seen broken when clickhouse defaults to 1970 --- .../overview/overview-metric-card.tsx | 64 +++++++-------- .../components/profiles/profile-metrics.tsx | 17 ++-- packages/common/src/date.ts | 4 +- packages/db/src/clickhouse/client.ts | 5 ++ packages/db/src/services/overview.service.ts | 78 +++++++++---------- packages/db/src/services/profile.service.ts | 38 ++++----- 6 files changed, 102 insertions(+), 104 deletions(-) diff --git a/apps/start/src/components/overview/overview-metric-card.tsx b/apps/start/src/components/overview/overview-metric-card.tsx index 66cbdfd7..a2ea8310 100644 --- a/apps/start/src/components/overview/overview-metric-card.tsx +++ b/apps/start/src/components/overview/overview-metric-card.tsx @@ -1,23 +1,16 @@ -import { fancyMinutes, useNumber } from '@/hooks/use-numer-formatter'; -import { cn } from '@/utils/cn'; -import AutoSizer from 'react-virtualized-auto-sizer'; -import { Area, AreaChart, Bar, BarChart, Tooltip } from 'recharts'; - -import { formatDate, timeAgo } from '@/utils/date'; -import { getChartColor } from '@/utils/theme'; import { getPreviousMetric } from '@openpanel/common'; import { useEffect, useRef, useState } from 'react'; +import AutoSizer from 'react-virtualized-auto-sizer'; +import { Bar, BarChart, Tooltip } from 'recharts'; import { - ChartTooltipContainer, - ChartTooltipHeader, - ChartTooltipItem, -} from '../charts/chart-tooltip'; -import { - PreviousDiffIndicatorPure, getDiffIndicator, + PreviousDiffIndicatorPure, } from '../report-chart/common/previous-diff-indicator'; import { Skeleton } from '../skeleton'; import { Tooltiper } from '../ui/tooltip'; +import { fancyMinutes, useNumber } from '@/hooks/use-numer-formatter'; +import { cn } from '@/utils/cn'; +import { formatDate, timeAgo } from '@/utils/date'; interface MetricCardProps { id: string; @@ -78,6 +71,9 @@ export function OverviewMetricCard({ } if (unit === 'timeAgo') { + if (!value) { + return <>{'N/A'}; + } return <>{timeAgo(new Date(value))}; } @@ -103,7 +99,7 @@ export function OverviewMetricCard({ getPreviousMetric(current, previous)?.state, '#6ee7b7', // green '#fda4af', // red - '#93c5fd', // blue + '#93c5fd' // blue ); const renderTooltip = () => { @@ -115,7 +111,7 @@ export function OverviewMetricCard({ {renderValue( data[currentIndex].current, 'ml-1 font-light text-xl', - false, + false )} @@ -132,60 +128,60 @@ export function OverviewMetricCard({ ); }; return ( - + @@ -207,9 +203,9 @@ export function OverviewMetricCardNumber({ isLoading?: boolean; }) { return ( -
+
- + {label}
@@ -219,11 +215,11 @@ export function OverviewMetricCardNumber({
) : ( -
+
{value}
)} -
+
{enhancer}
diff --git a/apps/start/src/components/profiles/profile-metrics.tsx b/apps/start/src/components/profiles/profile-metrics.tsx index eb30c53c..6558c4ee 100644 --- a/apps/start/src/components/profiles/profile-metrics.tsx +++ b/apps/start/src/components/profiles/profile-metrics.tsx @@ -1,6 +1,5 @@ -import { OverviewMetricCard } from '@/components/overview/overview-metric-card'; - import type { IProfileMetrics } from '@openpanel/db'; +import { OverviewMetricCard } from '@/components/overview/overview-metric-card'; type Props = { data: IProfileMetrics; @@ -102,7 +101,7 @@ const PROFILE_METRICS = [ export const ProfileMetrics = ({ data }: Props) => { return ( -
+
{PROFILE_METRICS.filter((metric) => { if (metric.hideOnZero && data[metric.key] === 0) { @@ -111,20 +110,20 @@ export const ProfileMetrics = ({ data }: Props) => { return true; }).map((metric) => ( ))}
diff --git a/packages/common/src/date.ts b/packages/common/src/date.ts index a6c32065..6a69d9cc 100644 --- a/packages/common/src/date.ts +++ b/packages/common/src/date.ts @@ -1,6 +1,6 @@ -import { DateTime } from 'luxon'; +export { DateTime } from 'luxon'; -export { DateTime }; +export type { DateTime }; export function getTime(date: string | number | Date) { return new Date(date).getTime(); diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts index c1d306a2..d2899f82 100644 --- a/packages/db/src/clickhouse/client.ts +++ b/packages/db/src/clickhouse/client.ts @@ -290,3 +290,8 @@ export function toDate(str: string, interval?: IInterval) { export function convertClickhouseDateToJs(date: string) { return new Date(`${date.replace(' ', 'T')}Z`); } + +const ROLLUP_DATE_PREFIX = '1970-01-01'; +export function isClickhouseDefaultMinDate(date: string): boolean { + return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31'); +} diff --git a/packages/db/src/services/overview.service.ts b/packages/db/src/services/overview.service.ts index c0619a2b..17b8d900 100644 --- a/packages/db/src/services/overview.service.ts +++ b/packages/db/src/services/overview.service.ts @@ -1,14 +1,13 @@ import { average, sum } from '@openpanel/common'; import { chartColors } from '@openpanel/constants'; -import { getCache } from '@openpanel/redis'; import { type IChartEventFilter, zTimeInterval } from '@openpanel/validation'; -import { omit } from 'ramda'; import sqlstring from 'sqlstring'; import { z } from 'zod'; import { - TABLE_NAMES, ch, convertClickhouseDateToJs, + isClickhouseDefaultMinDate, + TABLE_NAMES, } from '../clickhouse/client'; import { clix } from '../clickhouse/query-builder'; import { @@ -172,24 +171,12 @@ export type IGetMapDataInput = z.infer & { export class OverviewService { constructor(private client: typeof ch) {} - // Helper methods - private isRollupRow(date: string): boolean { - // The rollup row has date 1970-01-01 00:00:00 (epoch) from ClickHouse. - // After transform with `new Date().toISOString()`, this becomes an ISO string. - // Due to timezone handling in JavaScript's Date constructor (which interprets - // the input as local time), the UTC date might become: - // - 1969-12-31T... for positive UTC offsets (e.g., UTC+8) - // - 1970-01-01T... for UTC or negative offsets - // We check for both year prefixes to handle all server timezones. - return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31'); - } - private getFillConfig(interval: string, startDate: string, endDate: string) { const useDateOnly = ['month', 'week'].includes(interval); return { from: clix.toStartOf( clix.datetime(startDate, useDateOnly ? 'toDate' : 'toDateTime'), - interval as any, + interval as any ), to: clix.datetime(endDate, useDateOnly ? 'toDate' : 'toDateTime'), step: clix.toInterval('1', interval as any), @@ -234,12 +221,12 @@ export class OverviewService { private mergeRevenueIntoSeries( series: T[], - revenueData: { date: string; total_revenue: number }[], + revenueData: { date: string; total_revenue: number }[] ): (T & { total_revenue: number })[] { const revenueByDate = new Map( revenueData - .filter((r) => !this.isRollupRow(r.date)) - .map((r) => [r.date, r.total_revenue]), + .filter((r) => !isClickhouseDefaultMinDate(r.date)) + .map((r) => [r.date, r.total_revenue]) ); return series.map((row) => ({ ...row, @@ -248,10 +235,11 @@ export class OverviewService { } private getOverallRevenue( - revenueData: { date: string; total_revenue: number }[], + revenueData: { date: string; total_revenue: number }[] ): number { return ( - revenueData.find((r) => this.isRollupRow(r.date))?.total_revenue ?? 0 + revenueData.find((r) => isClickhouseDefaultMinDate(r.date)) + ?.total_revenue ?? 0 ); } @@ -263,7 +251,7 @@ export class OverviewService { startDate: string; endDate: string; timezone: string; - }, + } ): ReturnType { if (!this.isPageFilter(params.filters)) { query.rawWhere(this.getRawWhereClause('sessions', params.filters)); @@ -276,7 +264,7 @@ export class OverviewService { .where( 'id', 'IN', - clix.exp('(SELECT session_id FROM distinct_sessions)'), + clix.exp('(SELECT session_id FROM distinct_sessions)') ); } @@ -475,14 +463,14 @@ export class OverviewService { clix(this.client, timezone) .select(['bounce_rate']) .from('session_agg') - .where('date', '=', rollupDate), + .where('date', '=', rollupDate) ) .with( 'daily_session_stats', clix(this.client, timezone) .select(['date', 'bounce_rate']) .from('session_agg') - .where('date', '!=', rollupDate), + .where('date', '!=', rollupDate) ) .with('overall_unique_visitors', overallUniqueVisitorsQuery) .select<{ @@ -512,7 +500,7 @@ export class OverviewService { .from(`${TABLE_NAMES.events} AS e`) .leftJoin( 'daily_session_stats AS dss', - `${clix.toStartOf('e.created_at', interval as any)} = dss.date`, + `${clix.toStartOf('e.created_at', interval as any)} = dss.date` ) .where('e.project_id', '=', projectId) .where('e.name', '=', 'screen_view') @@ -551,7 +539,7 @@ export class OverviewService { (item) => item.overall_bounce_rate !== null || item.overall_total_sessions !== null || - item.overall_unique_visitors !== null, + item.overall_unique_visitors !== null ); return { @@ -560,11 +548,11 @@ export class OverviewService { unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0, total_sessions: anyRowWithData?.overall_total_sessions ?? 0, avg_session_duration: average( - mainRes.map((item) => item.avg_session_duration), + mainRes.map((item) => item.avg_session_duration) ), total_screen_views: sum(mainRes.map((item) => item.total_screen_views)), views_per_session: average( - mainRes.map((item) => item.views_per_session), + mainRes.map((item) => item.views_per_session) ), total_revenue: overallRevenue, }, @@ -591,7 +579,7 @@ export class OverviewService { return item; } return item; - }), + }) ); return Object.values(where).join(' AND '); @@ -879,7 +867,7 @@ export class OverviewService { startDate, endDate, timezone, - }, + } ); const timeSeriesData = await mainTimeSeriesQuery.execute(); @@ -1066,7 +1054,7 @@ export class OverviewService { .from('paths_deduped_cte') .having('length(paths)', '>=', 2) // ONLY sessions starting with top entry pages - .having('paths[1]', 'IN', topEntryPages), + .having('paths[1]', 'IN', topEntryPages) ) .select<{ source: string; @@ -1081,8 +1069,8 @@ export class OverviewService { ]) .from( clix.exp( - '(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)', - ), + '(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)' + ) ) .groupBy(['source', 'target', 'step']) .orderBy('step', 'ASC') @@ -1143,7 +1131,9 @@ export class OverviewService { for (const t of fromSource) { // Skip self-loops - if (t.source === t.target) continue; + if (t.source === t.target) { + continue; + } const targetNodeId = getNodeId(t.target, step + 1); @@ -1180,7 +1170,9 @@ export class OverviewService { } // Stop if no more nodes to process - if (activeNodes.size === 0) break; + if (activeNodes.size === 0) { + break; + } } // Step 5: Filter links by threshold (0.25% of total sessions) @@ -1235,22 +1227,24 @@ export class OverviewService { }) .sort((a, b) => { // Sort by step first, then by value descending - if (a.step !== b.step) return a.step - b.step; + if (a.step !== b.step) { + return a.step - b.step; + } return b.value - a.value; }); // Sanity check: Ensure all link endpoints exist in nodes const nodeIds = new Set(finalNodes.map((n) => n.id)); const invalidLinks = filteredLinks.filter( - (link) => !nodeIds.has(link.source) || !nodeIds.has(link.target), + (link) => !(nodeIds.has(link.source) && nodeIds.has(link.target)) ); if (invalidLinks.length > 0) { console.warn( - `UserJourney: Found ${invalidLinks.length} links with missing nodes`, + `UserJourney: Found ${invalidLinks.length} links with missing nodes` ); // Remove invalid links const validLinks = filteredLinks.filter( - (link) => nodeIds.has(link.source) && nodeIds.has(link.target), + (link) => nodeIds.has(link.source) && nodeIds.has(link.target) ); return { nodes: finalNodes, @@ -1260,7 +1254,9 @@ export class OverviewService { // Sanity check: Ensure steps are monotonic (should always be true, but verify) const stepsValid = finalNodes.every((node, idx, arr) => { - if (idx === 0) return true; + if (idx === 0) { + return true; + } return node.step! >= arr[idx - 1]!.step!; }); if (!stepsValid) { diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.ts index 41358791..424382b4 100644 --- a/packages/db/src/services/profile.service.ts +++ b/packages/db/src/services/profile.service.ts @@ -1,23 +1,21 @@ -import { omit, uniq } from 'ramda'; -import sqlstring from 'sqlstring'; - import { strip, toObject } from '@openpanel/common'; import { cacheable } from '@openpanel/redis'; import type { IChartEventFilter } from '@openpanel/validation'; - +import { uniq } from 'ramda'; +import sqlstring from 'sqlstring'; import { profileBuffer } from '../buffers'; import { - TABLE_NAMES, - ch, chQuery, convertClickhouseDateToJs, formatClickhouseDate, + isClickhouseDefaultMinDate, + TABLE_NAMES, } from '../clickhouse/client'; import { createSqlBuilder } from '../sql-builder'; -export type IProfileMetrics = { - lastSeen: Date; - firstSeen: Date; +export interface IProfileMetrics { + lastSeen: Date | null; + firstSeen: Date | null; screenViews: number; sessions: number; durationAvg: number; @@ -29,7 +27,7 @@ export type IProfileMetrics = { conversionEvents: number; avgTimeBetweenSessions: number; revenue: number; -}; +} export function getProfileMetrics(profileId: string, projectId: string) { return chQuery< Omit & { @@ -100,8 +98,12 @@ export function getProfileMetrics(profileId: string, projectId: string) { .then((data) => { return { ...data, - lastSeen: convertClickhouseDateToJs(data.lastSeen), - firstSeen: convertClickhouseDateToJs(data.firstSeen), + lastSeen: isClickhouseDefaultMinDate(data.lastSeen) + ? null + : convertClickhouseDateToJs(data.lastSeen), + firstSeen: isClickhouseDefaultMinDate(data.firstSeen) + ? null + : convertClickhouseDateToJs(data.firstSeen), }; }); } @@ -127,7 +129,7 @@ export async function getProfileById(id: string, projectId: string) { last_value(is_external) as is_external, last_value(properties) as properties, last_value(created_at) as created_at - FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`, + FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1` ); if (!profile) { @@ -169,7 +171,7 @@ export async function getProfiles(ids: string[], projectId: string) { project_id = ${sqlstring.escape(projectId)} AND id IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')}) GROUP BY id, project_id - `, + ` ); return data.map(transformProfile); @@ -221,7 +223,7 @@ export async function getProfileListCount({ return data[0]?.count ?? 0; } -export type IServiceProfile = { +export interface IServiceProfile { id: string; email: string; avatar: string; @@ -245,7 +247,7 @@ export type IServiceProfile = { model?: string; referrer?: string; }; -}; +} export interface IClickhouseProfile { id: string; @@ -289,7 +291,7 @@ export function transformProfile({ }; } -export async function upsertProfile( +export function upsertProfile( { id, firstName, @@ -300,7 +302,7 @@ export async function upsertProfile( projectId, isExternal, }: IServiceUpsertProfile, - isFromEvent = false, + isFromEvent = false ) { const profile: IClickhouseProfile = { id,