zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: default last/first seen broken when clickhouse defaults to 1970

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-02 09:34:23 +01:00
parent 647ac2a4af
commit 8c377c2066
6 changed files with 102 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
apps/start/src/components/overview/overview-metric-card.tsx
View File

@@ -1,23 +1,16 @@
import { fancyMinutes, useNumber } from '@/hooks/use-numer-formatter';
import { cn } from '@/utils/cn';
import AutoSizer from 'react-virtualized-auto-sizer';
import { Area, AreaChart, Bar, BarChart, Tooltip } from 'recharts';
import { formatDate, timeAgo } from '@/utils/date';
import { getChartColor } from '@/utils/theme';
import { getPreviousMetric } from '@openpanel/common';
import { useEffect, useRef, useState } from 'react';
import AutoSizer from 'react-virtualized-auto-sizer';
import { Bar, BarChart, Tooltip } from 'recharts';
import {
ChartTooltipContainer,
ChartTooltipHeader,
ChartTooltipItem,
} from '../charts/chart-tooltip';
import {
PreviousDiffIndicatorPure,
getDiffIndicator,
PreviousDiffIndicatorPure,
} from '../report-chart/common/previous-diff-indicator';
import { Skeleton } from '../skeleton';
import { Tooltiper } from '../ui/tooltip';
import { fancyMinutes, useNumber } from '@/hooks/use-numer-formatter';
import { cn } from '@/utils/cn';
import { formatDate, timeAgo } from '@/utils/date';
interface MetricCardProps {
id: string;
@@ -78,6 +71,9 @@ export function OverviewMetricCard({
}
if (unit === 'timeAgo') {
if (!value) {
return <>{'N/A'}</>;
}
return <>{timeAgo(new Date(value))}</>;
}
@@ -103,7 +99,7 @@ export function OverviewMetricCard({
getPreviousMetric(current, previous)?.state,
'#6ee7b7', // green
'#fda4af', // red
'#93c5fd', // blue
'#93c5fd' // blue
);
const renderTooltip = () => {
@@ -115,7 +111,7 @@ export function OverviewMetricCard({
{renderValue(
data[currentIndex].current,
'ml-1 font-light text-xl',
false,
false
)}
</span>
</span>
@@ -132,60 +128,60 @@ export function OverviewMetricCard({
);
};
return (
<Tooltiper content={renderTooltip()} asChild sideOffset={-20}>
<Tooltiper asChild content={renderTooltip()} sideOffset={-20}>
<button
type="button"
className={cn(
'col-span-2 flex-1 shadow-[0_0_0_0.5px] shadow-border md:col-span-1',
active && 'bg-def-100',
active && 'bg-def-100'
)}
onClick={onClick}
type="button"
>
<div className={cn('group relative p-4')}>
<div
className={cn(
'absolute left-4 right-4 bottom-0 z-0 opacity-50 transition-opacity duration-300 group-hover:opacity-100',
'absolute right-4 bottom-0 left-4 z-0 opacity-50 transition-opacity duration-300 group-hover:opacity-100'
)}
>
<AutoSizer style={{ height: 20 }}>
{({ width }) => (
<BarChart
width={width}
height={20}
data={data}
style={{
background: 'transparent',
}}
height={20}
margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
onMouseMove={(event) => {
setCurrentIndex(event.activeTooltipIndex ?? null);
}}
margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
style={{
background: 'transparent',
}}
width={width}
>
<Tooltip content={() => null} cursor={false} />
<Bar
dataKey={'current'}
type="step"
fill={graphColors}
fillOpacity={1}
strokeWidth={0}
isAnimationActive={false}
strokeWidth={0}
type="step"
/>
</BarChart>
)}
</AutoSizer>
</div>
<OverviewMetricCardNumber
label={label}
value={renderValue(current, 'ml-1 font-light text-xl')}
enhancer={
<PreviousDiffIndicatorPure
className="text-sm"
size="sm"
inverted={inverted}
size="sm"
{...getPreviousMetric(current, previous)}
/>
}
isLoading={isLoading}
label={label}
value={renderValue(current, 'ml-1 font-light text-xl')}
/>
</div>
</button>
@@ -207,9 +203,9 @@ export function OverviewMetricCardNumber({
isLoading?: boolean;
}) {
return (
<div className={cn('min-w-0 col gap-2', className)}>
<div className={cn('col min-w-0 gap-2', className)}>
<div className="flex min-w-0 items-center gap-2 text-left">
<span className="truncate text-sm font-medium text-muted-foreground leading-[1.1]">
<span className="truncate font-medium text-muted-foreground text-sm leading-[1.1]">
{label}
</span>
</div>
@@ -219,11 +215,11 @@ export function OverviewMetricCardNumber({
<Skeleton className="h-6 w-12" />
</div>
) : (
<div className="truncate font-mono text-3xl leading-[1.1] font-bold w-full text-left">
<div className="w-full truncate text-left font-bold font-mono text-3xl leading-[1.1]">
{value}
</div>
)}
<div className="absolute right-0 top-0 bottom-0 center justify-center col pr-4">
<div className="center col absolute top-0 right-0 bottom-0 justify-center pr-4">
{enhancer}
</div>
</div>

17
apps/start/src/components/profiles/profile-metrics.tsx
View File

@@ -1,6 +1,5 @@
import { OverviewMetricCard } from '@/components/overview/overview-metric-card';
import type { IProfileMetrics } from '@openpanel/db';
import { OverviewMetricCard } from '@/components/overview/overview-metric-card';
type Props = {
data: IProfileMetrics;
@@ -102,7 +101,7 @@ const PROFILE_METRICS = [
export const ProfileMetrics = ({ data }: Props) => {
return (
<div className="relative col-span-6 -m-4 mb-0 mt-0 md:m-0">
<div className="relative col-span-6 -m-4 mt-0 mb-0 md:m-0">
<div className="card grid grid-cols-2 overflow-hidden rounded-md md:grid-cols-4 lg:grid-cols-6">
{PROFILE_METRICS.filter((metric) => {
if (metric.hideOnZero && data[metric.key] === 0) {
@@ -111,20 +110,20 @@ export const ProfileMetrics = ({ data }: Props) => {
return true;
}).map((metric) => (
<OverviewMetricCard
key={metric.key}
data={[]}
id={metric.key}
inverted={metric.inverted}
isLoading={false}
key={metric.key}
label={metric.title}
metric={{
current:
metric.unit === 'timeAgo'
? new Date(data[metric.key]).getTime()
metric.unit === 'timeAgo' && data[metric.key]
? new Date(data[metric.key]!).getTime()
: (data[metric.key] as number) || 0,
previous: null,
}}
unit={metric.unit}
data={[]}
inverted={metric.inverted}
isLoading={false}
/>
))}
</div>

4
packages/common/src/date.ts
View File

@@ -1,6 +1,6 @@
import { DateTime } from 'luxon';
export { DateTime } from 'luxon';
export { DateTime };
export type { DateTime };
export function getTime(date: string | number | Date) {
return new Date(date).getTime();

5
packages/db/src/clickhouse/client.ts
View File

@@ -290,3 +290,8 @@ export function toDate(str: string, interval?: IInterval) {
export function convertClickhouseDateToJs(date: string) {
return new Date(`${date.replace(' ', 'T')}Z`);
}
const ROLLUP_DATE_PREFIX = '1970-01-01';
export function isClickhouseDefaultMinDate(date: string): boolean {
return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');
}

78
packages/db/src/services/overview.service.ts
View File

@@ -1,14 +1,13 @@
import { average, sum } from '@openpanel/common';
import { chartColors } from '@openpanel/constants';
import { getCache } from '@openpanel/redis';
import { type IChartEventFilter, zTimeInterval } from '@openpanel/validation';
import { omit } from 'ramda';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import {
TABLE_NAMES,
ch,
convertClickhouseDateToJs,
isClickhouseDefaultMinDate,
TABLE_NAMES,
} from '../clickhouse/client';
import { clix } from '../clickhouse/query-builder';
import {
@@ -172,24 +171,12 @@ export type IGetMapDataInput = z.infer<typeof zGetMapDataInput> & {
export class OverviewService {
constructor(private client: typeof ch) {}
// Helper methods
private isRollupRow(date: string): boolean {
// The rollup row has date 1970-01-01 00:00:00 (epoch) from ClickHouse.
// After transform with `new Date().toISOString()`, this becomes an ISO string.
// Due to timezone handling in JavaScript's Date constructor (which interprets
// the input as local time), the UTC date might become:
// - 1969-12-31T... for positive UTC offsets (e.g., UTC+8)
// - 1970-01-01T... for UTC or negative offsets
// We check for both year prefixes to handle all server timezones.
return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');
}
private getFillConfig(interval: string, startDate: string, endDate: string) {
const useDateOnly = ['month', 'week'].includes(interval);
return {
from: clix.toStartOf(
clix.datetime(startDate, useDateOnly ? 'toDate' : 'toDateTime'),
interval as any,
interval as any
),
to: clix.datetime(endDate, useDateOnly ? 'toDate' : 'toDateTime'),
step: clix.toInterval('1', interval as any),
@@ -234,12 +221,12 @@ export class OverviewService {
private mergeRevenueIntoSeries<T extends { date: string }>(
series: T[],
revenueData: { date: string; total_revenue: number }[],
revenueData: { date: string; total_revenue: number }[]
): (T & { total_revenue: number })[] {
const revenueByDate = new Map(
revenueData
.filter((r) => !this.isRollupRow(r.date))
.map((r) => [r.date, r.total_revenue]),
.filter((r) => !isClickhouseDefaultMinDate(r.date))
.map((r) => [r.date, r.total_revenue])
);
return series.map((row) => ({
...row,
@@ -248,10 +235,11 @@ export class OverviewService {
}
private getOverallRevenue(
revenueData: { date: string; total_revenue: number }[],
revenueData: { date: string; total_revenue: number }[]
): number {
return (
revenueData.find((r) => this.isRollupRow(r.date))?.total_revenue ?? 0
revenueData.find((r) => isClickhouseDefaultMinDate(r.date))
?.total_revenue ?? 0
);
}
@@ -263,7 +251,7 @@ export class OverviewService {
startDate: string;
endDate: string;
timezone: string;
},
}
): ReturnType<typeof clix> {
if (!this.isPageFilter(params.filters)) {
query.rawWhere(this.getRawWhereClause('sessions', params.filters));
@@ -276,7 +264,7 @@ export class OverviewService {
.where(
'id',
'IN',
clix.exp('(SELECT session_id FROM distinct_sessions)'),
clix.exp('(SELECT session_id FROM distinct_sessions)')
);
}
@@ -475,14 +463,14 @@ export class OverviewService {
clix(this.client, timezone)
.select(['bounce_rate'])
.from('session_agg')
.where('date', '=', rollupDate),
.where('date', '=', rollupDate)
)
.with(
'daily_session_stats',
clix(this.client, timezone)
.select(['date', 'bounce_rate'])
.from('session_agg')
.where('date', '!=', rollupDate),
.where('date', '!=', rollupDate)
)
.with('overall_unique_visitors', overallUniqueVisitorsQuery)
.select<{
@@ -512,7 +500,7 @@ export class OverviewService {
.from(`${TABLE_NAMES.events} AS e`)
.leftJoin(
'daily_session_stats AS dss',
`${clix.toStartOf('e.created_at', interval as any)} = dss.date`,
`${clix.toStartOf('e.created_at', interval as any)} = dss.date`
)
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
@@ -551,7 +539,7 @@ export class OverviewService {
(item) =>
item.overall_bounce_rate !== null ||
item.overall_total_sessions !== null ||
item.overall_unique_visitors !== null,
item.overall_unique_visitors !== null
);
return {
@@ -560,11 +548,11 @@ export class OverviewService {
unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0,
total_sessions: anyRowWithData?.overall_total_sessions ?? 0,
avg_session_duration: average(
mainRes.map((item) => item.avg_session_duration),
mainRes.map((item) => item.avg_session_duration)
),
total_screen_views: sum(mainRes.map((item) => item.total_screen_views)),
views_per_session: average(
mainRes.map((item) => item.views_per_session),
mainRes.map((item) => item.views_per_session)
),
total_revenue: overallRevenue,
},
@@ -591,7 +579,7 @@ export class OverviewService {
return item;
}
return item;
}),
})
);
return Object.values(where).join(' AND ');
@@ -879,7 +867,7 @@ export class OverviewService {
startDate,
endDate,
timezone,
},
}
);
const timeSeriesData = await mainTimeSeriesQuery.execute();
@@ -1066,7 +1054,7 @@ export class OverviewService {
.from('paths_deduped_cte')
.having('length(paths)', '>=', 2)
// ONLY sessions starting with top entry pages
.having('paths[1]', 'IN', topEntryPages),
.having('paths[1]', 'IN', topEntryPages)
)
.select<{
source: string;
@@ -1081,8 +1069,8 @@ export class OverviewService {
])
.from(
clix.exp(
'(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)',
),
'(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)'
)
)
.groupBy(['source', 'target', 'step'])
.orderBy('step', 'ASC')
@@ -1143,7 +1131,9 @@ export class OverviewService {
for (const t of fromSource) {
// Skip self-loops
if (t.source === t.target) continue;
if (t.source === t.target) {
continue;
}
const targetNodeId = getNodeId(t.target, step + 1);
@@ -1180,7 +1170,9 @@ export class OverviewService {
}
// Stop if no more nodes to process
if (activeNodes.size === 0) break;
if (activeNodes.size === 0) {
break;
}
}
// Step 5: Filter links by threshold (0.25% of total sessions)
@@ -1235,22 +1227,24 @@ export class OverviewService {
})
.sort((a, b) => {
// Sort by step first, then by value descending
if (a.step !== b.step) return a.step - b.step;
if (a.step !== b.step) {
return a.step - b.step;
}
return b.value - a.value;
});
// Sanity check: Ensure all link endpoints exist in nodes
const nodeIds = new Set(finalNodes.map((n) => n.id));
const invalidLinks = filteredLinks.filter(
(link) => !nodeIds.has(link.source) || !nodeIds.has(link.target),
(link) => !(nodeIds.has(link.source) && nodeIds.has(link.target))
);
if (invalidLinks.length > 0) {
console.warn(
`UserJourney: Found ${invalidLinks.length} links with missing nodes`,
`UserJourney: Found ${invalidLinks.length} links with missing nodes`
);
// Remove invalid links
const validLinks = filteredLinks.filter(
(link) => nodeIds.has(link.source) && nodeIds.has(link.target),
(link) => nodeIds.has(link.source) && nodeIds.has(link.target)
);
return {
nodes: finalNodes,
@@ -1260,7 +1254,9 @@ export class OverviewService {
// Sanity check: Ensure steps are monotonic (should always be true, but verify)
const stepsValid = finalNodes.every((node, idx, arr) => {
if (idx === 0) return true;
if (idx === 0) {
return true;
}
return node.step! >= arr[idx - 1]!.step!;
});
if (!stepsValid) {

38
packages/db/src/services/profile.service.ts
View File

@@ -1,23 +1,21 @@
import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { strip, toObject } from '@openpanel/common';
import { cacheable } from '@openpanel/redis';
import type { IChartEventFilter } from '@openpanel/validation';
import { uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { profileBuffer } from '../buffers';
import {
TABLE_NAMES,
ch,
chQuery,
convertClickhouseDateToJs,
formatClickhouseDate,
isClickhouseDefaultMinDate,
TABLE_NAMES,
} from '../clickhouse/client';
import { createSqlBuilder } from '../sql-builder';
export type IProfileMetrics = {
lastSeen: Date;
firstSeen: Date;
export interface IProfileMetrics {
lastSeen: Date | null;
firstSeen: Date | null;
screenViews: number;
sessions: number;
durationAvg: number;
@@ -29,7 +27,7 @@ export type IProfileMetrics = {
conversionEvents: number;
avgTimeBetweenSessions: number;
revenue: number;
};
}
export function getProfileMetrics(profileId: string, projectId: string) {
return chQuery<
Omit<IProfileMetrics, 'lastSeen' | 'firstSeen'> & {
@@ -100,8 +98,12 @@ export function getProfileMetrics(profileId: string, projectId: string) {
.then((data) => {
return {
...data,
lastSeen: convertClickhouseDateToJs(data.lastSeen),
firstSeen: convertClickhouseDateToJs(data.firstSeen),
lastSeen: isClickhouseDefaultMinDate(data.lastSeen)
? null
: convertClickhouseDateToJs(data.lastSeen),
firstSeen: isClickhouseDefaultMinDate(data.firstSeen)
? null
: convertClickhouseDateToJs(data.firstSeen),
};
});
}
@@ -127,7 +129,7 @@ export async function getProfileById(id: string, projectId: string) {
last_value(is_external) as is_external,
last_value(properties) as properties,
last_value(created_at) as created_at
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`
);
if (!profile) {
@@ -169,7 +171,7 @@ export async function getProfiles(ids: string[], projectId: string) {
project_id = ${sqlstring.escape(projectId)} AND
id IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')})
GROUP BY id, project_id
`,
`
);
return data.map(transformProfile);
@@ -221,7 +223,7 @@ export async function getProfileListCount({
return data[0]?.count ?? 0;
}
export type IServiceProfile = {
export interface IServiceProfile {
id: string;
email: string;
avatar: string;
@@ -245,7 +247,7 @@ export type IServiceProfile = {
model?: string;
referrer?: string;
};
};
}
export interface IClickhouseProfile {
id: string;
@@ -289,7 +291,7 @@ export function transformProfile({
};
}
export async function upsertProfile(
export function upsertProfile(
{
id,
firstName,
@@ -300,7 +302,7 @@ export async function upsertProfile(
projectId,
isExternal,
}: IServiceUpsertProfile,
isFromEvent = false,
isFromEvent = false
) {
const profile: IClickhouseProfile = {
id,