zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:buffers

Browse Source 
* wip

* remove active visitor counter in redis

* test

* fix profiel query

* fix
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-24 13:54:00 +01:00
committed by GitHub
parent 20665789e1
commit a1ce71ffb6
11 changed files with 690 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
apps/api/src/controllers/live.controller.ts
View File

@@ -1,10 +1,7 @@
import type { WebSocket } from '@fastify/websocket';
import { eventBuffer } from '@openpanel/db';
import { setSuperJson } from '@openpanel/json';
import {
psubscribeToPublishedEvent,
subscribeToPublishedEvent,
} from '@openpanel/redis';
import { subscribeToPublishedEvent } from '@openpanel/redis';
import { getProjectAccess } from '@openpanel/trpc';
import { getOrganizationAccess } from '@openpanel/trpc/src/access';
import type { FastifyRequest } from 'fastify';
@@ -39,19 +36,8 @@ export function wsVisitors(
}
);
const punsubscribe = psubscribeToPublishedEvent(
'__keyevent@0__:expired',
(key) => {
const [, , projectId] = key.split(':');
if (projectId === params.projectId) {
sendCount();
}
}
);
socket.on('close', () => {
unsubscribe();
punsubscribe();
});
}

66
apps/start/src/components/overview/live-counter.tsx
View File

@@ -1,61 +1,25 @@
import { TooltipComplete } from '@/components/tooltip-complete';
import { useDebounceState } from '@/hooks/use-debounce-state';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import { cn } from '@/utils/cn';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef } from 'react';
import { useQueryClient } from '@tanstack/react-query';
import { useCallback } from 'react';
import { toast } from 'sonner';
import { AnimatedNumber } from '../animated-number';
import { TooltipComplete } from '@/components/tooltip-complete';
import { useLiveCounter } from '@/hooks/use-live-counter';
import { cn } from '@/utils/cn';
export interface LiveCounterProps {
projectId: string;
shareId?: string;
}
const FIFTEEN_SECONDS = 1000 * 30;
export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
const trpc = useTRPC();
const client = useQueryClient();
const counter = useDebounceState(0, 1000);
const lastRefresh = useRef(Date.now());
const query = useQuery(
trpc.overview.liveVisitors.queryOptions({
projectId,
shareId,
}),
);
useEffect(() => {
if (query.data) {
counter.set(query.data);
}
}, [query.data]);
useWS<number>(
`/live/visitors/${projectId}`,
(value) => {
if (!Number.isNaN(value)) {
counter.set(value);
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
lastRefresh.current = Date.now();
if (!document.hidden) {
toast('Refreshed data');
client.refetchQueries({
type: 'active',
});
}
}
}
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
},
);
const onRefresh = useCallback(() => {
toast('Refreshed data');
client.refetchQueries({
type: 'active',
});
}, [client]);
const counter = useLiveCounter({ projectId, shareId, onRefresh });
return (
<TooltipComplete
@@ -66,13 +30,13 @@ export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
<div
className={cn(
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
counter.debounced === 0 && 'bg-destructive opacity-0',
counter.debounced === 0 && 'bg-destructive opacity-0'
)}
/>
<div
className={cn(
'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
counter.debounced === 0 && 'bg-destructive',
'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
counter.debounced === 0 && 'bg-destructive'
)}
/>
</div>

81
apps/start/src/hooks/use-live-counter.ts Normal file
View File

@@ -0,0 +1,81 @@
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef } from 'react';
import { useDebounceState } from './use-debounce-state';
import useWS from './use-ws';
import { useTRPC } from '@/integrations/trpc/react';
const FIFTEEN_SECONDS = 1000 * 15;
/** Refetch from API when WS-only updates may be stale (e.g. visitors left). */
const FALLBACK_STALE_MS = 1000 * 60;
export function useLiveCounter({
projectId,
shareId,
onRefresh,
}: {
projectId: string;
shareId?: string;
onRefresh?: () => void;
}) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const counter = useDebounceState(0, 1000);
const lastRefresh = useRef(Date.now());
const query = useQuery(
trpc.overview.liveVisitors.queryOptions({
projectId,
shareId: shareId ?? undefined,
})
);
useEffect(() => {
if (query.data) {
counter.set(query.data);
}
}, [query.data]);
useWS<number>(
`/live/visitors/${projectId}`,
(value) => {
if (!Number.isNaN(value)) {
counter.set(value);
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
lastRefresh.current = Date.now();
if (!document.hidden) {
onRefresh?.();
}
}
}
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
}
);
useEffect(() => {
const id = setInterval(async () => {
if (Date.now() - lastRefresh.current < FALLBACK_STALE_MS) {
return;
}
const data = await queryClient.fetchQuery(
trpc.overview.liveVisitors.queryOptions(
{
projectId,
shareId: shareId ?? undefined,
},
// Default query staleTime is 5m; bypass cache so this reconciliation always hits the API.
{ staleTime: 0 }
)
);
counter.set(data);
lastRefresh.current = Date.now();
}, FALLBACK_STALE_MS);
return () => clearInterval(id);
}, [projectId, shareId, trpc, queryClient, counter.set]);
return counter;
}

38
apps/start/src/routes/widget/counter.tsx
View File

@@ -1,12 +1,11 @@
import { AnimatedNumber } from '@/components/animated-number';
import { Ping } from '@/components/ping';
import { useNumber } from '@/hooks/use-numer-formatter';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import type { RouterOutputs } from '@/trpc/client';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number';
import { Ping } from '@/components/ping';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import type { RouterOutputs } from '@/trpc/client';
const widgetSearchSchema = z.object({
shareId: z.string(),
@@ -20,33 +19,33 @@ export const Route = createFileRoute('/widget/counter')({
});
function RouteComponent() {
const { shareId, limit, color } = Route.useSearch();
const { shareId } = Route.useSearch();
const trpc = useTRPC();
// Fetch widget data
const { data, isLoading } = useQuery(
trpc.widget.counter.queryOptions({ shareId }),
trpc.widget.counter.queryOptions({ shareId })
);
if (isLoading) {
return (
<div className="flex items-center gap-2 px-2 h-8">
<div className="flex h-8 items-center gap-2 px-2">
<Ping />
<AnimatedNumber value={0} suffix=" unique visitors" />
<AnimatedNumber suffix=" unique visitors" value={0} />
</div>
);
}
if (!data) {
return (
<div className="flex items-center gap-2 px-2 h-8">
<div className="flex h-8 items-center gap-2 px-2">
<Ping className="bg-orange-500" />
<AnimatedNumber value={0} suffix=" unique visitors" />
<AnimatedNumber suffix=" unique visitors" value={0} />
</div>
);
}
return <CounterWidget shareId={shareId} data={data} />;
return <CounterWidget data={data} shareId={shareId} />;
}
interface RealtimeWidgetProps {
@@ -57,30 +56,29 @@ interface RealtimeWidgetProps {
function CounterWidget({ shareId, data }: RealtimeWidgetProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const number = useNumber();
// WebSocket subscription for real-time updates
useWS<number>(
`/live/visitors/${data.projectId}`,
(res) => {
() => {
if (!document.hidden) {
queryClient.refetchQueries(
trpc.widget.counter.queryFilter({ shareId }),
trpc.widget.counter.queryFilter({ shareId })
);
}
},
{
debounce: {
delay: 1000,
maxWait: 60000,
maxWait: 60_000,
},
},
}
);
return (
<div className="flex items-center gap-2 px-2 h-8">
<div className="flex h-8 items-center gap-2 px-2">
<Ping />
<AnimatedNumber value={data.counter} suffix=" unique visitors" />
<AnimatedNumber suffix=" unique visitors" value={data.counter} />
</div>
);
}

150
apps/start/src/routes/widget/realtime.tsx
View File

@@ -1,3 +1,15 @@
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import type React from 'react';
import {
Bar,
BarChart,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number';
import {
ChartTooltipContainer,
@@ -14,18 +26,6 @@ import { countries } from '@/translations/countries';
import type { RouterOutputs } from '@/trpc/client';
import { cn } from '@/utils/cn';
import { getChartColor } from '@/utils/theme';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import type React from 'react';
import {
Bar,
BarChart,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
import { z } from 'zod';
const widgetSearchSchema = z.object({
shareId: z.string(),
@@ -44,7 +44,7 @@ function RouteComponent() {
// Fetch widget data
const { data: widgetData, isLoading } = useQuery(
trpc.widget.realtimeData.queryOptions({ shareId }),
trpc.widget.realtimeData.queryOptions({ shareId })
);
if (isLoading) {
@@ -53,10 +53,10 @@ function RouteComponent() {
if (!widgetData) {
return (
<div className="flex h-screen w-full center-center bg-background text-foreground col p-4">
<LogoSquare className="size-10 mb-4" />
<h1 className="text-xl font-semibold">Widget not found</h1>
<p className="mt-2 text-sm text-muted-foreground">
<div className="center-center col flex h-screen w-full bg-background p-4 text-foreground">
<LogoSquare className="mb-4 size-10" />
<h1 className="font-semibold text-xl">Widget not found</h1>
<p className="mt-2 text-muted-foreground text-sm">
This widget is not available or has been removed.
</p>
</div>
@@ -65,10 +65,10 @@ function RouteComponent() {
return (
<RealtimeWidget
shareId={shareId}
limit={limit}
data={widgetData}
color={color}
data={widgetData}
limit={limit}
shareId={shareId}
/>
);
}
@@ -83,7 +83,6 @@ interface RealtimeWidgetProps {
function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const number = useNumber();
// WebSocket subscription for real-time updates
useWS<number>(
@@ -91,16 +90,16 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
() => {
if (!document.hidden) {
queryClient.refetchQueries(
trpc.widget.realtimeData.queryFilter({ shareId }),
trpc.widget.realtimeData.queryFilter({ shareId })
);
}
},
{
debounce: {
delay: 1000,
maxWait: 60000,
maxWait: 60_000,
},
},
}
);
const maxDomain =
@@ -111,8 +110,12 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const referrers = data.referrers.length > 0 ? 1 : 0;
const paths = data.paths.length > 0 ? 1 : 0;
const value = countries + referrers + paths;
if (value === 3) return 'md:grid-cols-3';
if (value === 2) return 'md:grid-cols-2';
if (value === 3) {
return 'md:grid-cols-3';
}
if (value === 2) {
return 'md:grid-cols-2';
}
return 'md:grid-cols-1';
})();
@@ -120,10 +123,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
<div className="flex h-screen w-full flex-col bg-background text-foreground">
{/* Header with live counter */}
<div className="p-6 pb-3">
<div className="flex items-center justify-between w-full h-4">
<div className="flex items-center gap-3 w-full">
<div className="flex h-4 w-full items-center justify-between">
<div className="flex w-full items-center gap-3">
<Ping />
<div className="text-sm font-medium text-muted-foreground flex-1">
<div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES
</div>
{data.project.domain && <SerieIcon name={data.project.domain} />}
@@ -131,14 +134,14 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
</div>
<div className="row">
<div className="font-mono text-6xl font-bold h-18 text-foreground">
<div className="h-18 font-bold font-mono text-6xl text-foreground">
<AnimatedNumber value={data.liveCount} />
</div>
</div>
<div className="flex h-20 w-full flex-col -mt-4">
<div className="-mt-4 flex h-20 w-full flex-col">
<div className="flex-1">
<ResponsiveContainer width="100%" height="100%">
<ResponsiveContainer height="100%" width="100%">
<BarChart
data={data.histogram}
margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
@@ -148,22 +151,22 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
cursor={{ fill: 'var(--def-100)', radius: 4 }}
/>
<XAxis
dataKey="time"
axisLine={false}
tickLine={false}
dataKey="time"
interval="preserveStartEnd"
tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }}
tickLine={false}
ticks={[
data.histogram[0].time,
data.histogram[data.histogram.length - 1].time,
]}
interval="preserveStartEnd"
/>
<YAxis hide domain={[0, maxDomain]} />
<YAxis domain={[0, maxDomain]} hide />
<Bar
dataKey="sessionCount"
fill={color || 'var(--chart-0)'}
isAnimationActive={false}
radius={[4, 4, 4, 4]}
fill={color || 'var(--chart-0)'}
/>
</BarChart>
</ResponsiveContainer>
@@ -174,24 +177,24 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{(data.countries.length > 0 ||
data.referrers.length > 0 ||
data.paths.length > 0) && (
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar border-t">
<div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto border-t p-6">
<div className={cn('grid grid-cols-1 gap-6', grids)}>
{/* Countries */}
{data.countries.length > 0 && (
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.countries,
limit,
limit
);
return (
<>
{visible.map((item) => (
<RowItem key={item.country} count={item.count}>
<RowItem count={item.count} key={item.country}>
<div className="flex items-center gap-2">
<SerieIcon name={item.country} />
<span className="text-sm">
@@ -224,19 +227,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Referrers */}
{data.referrers.length > 0 && (
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.referrers,
limit,
limit
);
return (
<>
{visible.map((item) => (
<RowItem key={item.referrer} count={item.count}>
<RowItem count={item.count} key={item.referrer}>
<div className="flex items-center gap-2">
<SerieIcon name={item.referrer} />
<span className="truncate text-sm">
@@ -263,19 +266,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Paths */}
{data.paths.length > 0 && (
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
PATH
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.paths,
limit,
limit
);
return (
<>
{visible.map((item) => (
<RowItem key={item.path} count={item.count}>
<RowItem count={item.count} key={item.path}>
<span className="truncate text-sm">
{item.path}
</span>
@@ -303,10 +306,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
}
// Custom tooltip component that uses portals to escape overflow hidden
const CustomTooltip = ({ active, payload, coordinate }: any) => {
const CustomTooltip = ({ active, payload }: any) => {
const number = useNumber();
if (!active || !payload || !payload.length) {
if (!(active && payload && payload.length)) {
return null;
}
@@ -328,10 +331,13 @@ const CustomTooltip = ({ active, payload, coordinate }: any) => {
function RowItem({
children,
count,
}: { children: React.ReactNode; count: number }) {
}: {
children: React.ReactNode;
count: number;
}) {
const number = useNumber();
return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
{children}
<span className="font-semibold">{number.short(count)}</span>
</div>
@@ -340,7 +346,7 @@ function RowItem({
function getRestItems<T extends { count: number }>(
items: T[],
limit: number,
limit: number
): { visible: T[]; rest: T[]; restCount: number } {
const visible = items.slice(0, limit);
const rest = items.slice(limit);
@@ -375,7 +381,7 @@ function RestRow({
: 'paths';
return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
<span className="truncate">
{firstName} and {otherCount} more {typeLabel}...
</span>
@@ -434,13 +440,13 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
const itemCount = Math.min(limit, 5);
return (
<div className="flex h-screen w-full flex-col bg-background text-foreground animate-pulse">
<div className="flex h-screen w-full animate-pulse flex-col bg-background text-foreground">
{/* Header with live counter */}
<div className="border-b p-6 pb-3">
<div className="flex items-center justify-between w-full h-4">
<div className="flex items-center gap-3 w-full">
<div className="flex h-4 w-full items-center justify-between">
<div className="flex w-full items-center gap-3">
<div className="size-2 rounded-full bg-muted" />
<div className="text-sm font-medium text-muted-foreground flex-1">
<div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES
</div>
</div>
@@ -448,35 +454,35 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
</div>
<div className="row">
<div className="font-mono text-6xl font-bold h-18 flex items-center py-4 gap-1 row">
<div className="h-full w-6 bg-muted rounded" />
<div className="h-full w-6 bg-muted rounded" />
<div className="row flex h-18 items-center gap-1 py-4 font-bold font-mono text-6xl">
<div className="h-full w-6 rounded bg-muted" />
<div className="h-full w-6 rounded bg-muted" />
</div>
</div>
<div className="flex h-20 w-full flex-col -mt-4 pb-2.5">
<div className="flex-1 row gap-1 h-full">
<div className="-mt-4 flex h-20 w-full flex-col pb-2.5">
<div className="row h-full flex-1 gap-1">
{SKELETON_HISTOGRAM.map((item, index) => (
<div
className="mt-auto h-full w-full rounded bg-muted"
key={index.toString()}
style={{ height: `${item}%` }}
className="h-full w-full bg-muted rounded mt-auto"
/>
))}
</div>
<div className="row justify-between pt-2">
<div className="h-3 w-8 bg-muted rounded" />
<div className="h-3 w-8 bg-muted rounded" />
<div className="h-3 w-8 rounded bg-muted" />
<div className="h-3 w-8 rounded bg-muted" />
</div>
</div>
</div>
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar">
<div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto p-6">
{/* Countries, Referrers, and Paths skeleton */}
<div className="grid grid-cols-1 gap-6 md:grid-cols-3">
{/* Countries skeleton */}
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY
</div>
<div className="col">
@@ -488,7 +494,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Referrers skeleton */}
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER
</div>
<div className="col">
@@ -500,7 +506,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Paths skeleton */}
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
PATH
</div>
<div className="col">
@@ -517,12 +523,12 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
function RowItemSkeleton() {
return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b -mx-3">
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm">
<div className="flex items-center gap-2">
<div className="size-5 rounded bg-muted" />
<div className="h-4 w-24 bg-muted rounded" />
<div className="h-4 w-24 rounded bg-muted" />
</div>
<div className="h-4 w-8 bg-muted rounded" />
<div className="h-4 w-8 rounded bg-muted" />
</div>
);
}

44
packages/db/src/buffers/event-buffer.test.ts
View File

@@ -1,6 +1,7 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { ch } from '../clickhouse/client';
import * as chClient from '../clickhouse/client';
const { ch } = chClient;
// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
vi.mock('../services/event.service', () => ({}));
@@ -10,7 +11,8 @@ import { EventBuffer } from './event-buffer';
const redis = getRedisCache();
beforeEach(async () => {
await redis.flushdb();
const keys = await redis.keys('event*');
if (keys.length > 0) await redis.del(...keys);
});
afterAll(async () => {
@@ -209,18 +211,16 @@ describe('EventBuffer', () => {
});
it('tracks active visitors', async () => {
const event = {
project_id: 'p9',
profile_id: 'u9',
name: 'custom',
created_at: new Date().toISOString(),
} as any;
eventBuffer.add(event);
await eventBuffer.flush();
const querySpy = vi
.spyOn(chClient, 'chQuery')
.mockResolvedValueOnce([{ count: 2 }] as any);
const count = await eventBuffer.getActiveVisitorCount('p9');
expect(count).toBeGreaterThanOrEqual(1);
expect(count).toBe(2);
expect(querySpy).toHaveBeenCalledOnce();
expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");
querySpy.mockRestore();
});
it('handles multiple sessions independently — all events go to buffer', async () => {
@@ -273,4 +273,24 @@ describe('EventBuffer', () => {
expect(await eventBuffer.getBufferSize()).toBe(5);
});
it('retains events in queue when ClickHouse insert fails', async () => {
eventBuffer.add({
project_id: 'p12',
name: 'event1',
created_at: new Date().toISOString(),
} as any);
await eventBuffer.flush();
const insertSpy = vi
.spyOn(ch, 'insert')
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await eventBuffer.processBuffer();
// Events must still be in the queue — not lost
expect(await eventBuffer.getBufferSize()).toBe(1);
insertSpy.mockRestore();
});
});

71
packages/db/src/buffers/event-buffer.ts
View File

@@ -1,6 +1,6 @@
import { getSafeJson } from '@openpanel/json';
import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import { getRedisCache, publishEvent } from '@openpanel/redis';
import { ch, chQuery } from '../clickhouse/client';
import type { IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
@@ -25,10 +25,6 @@ export class EventBuffer extends BaseBuffer {
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
private activeVisitorsExpiration = 60 * 5; // 5 minutes
/** How often (ms) we refresh the heartbeat key + zadd per visitor. */
private heartbeatRefreshMs = 60_000; // 1 minute
private lastHeartbeat = new Map<string, number>();
private queueKey = 'event_buffer:queue';
protected bufferCounterKey = 'event_buffer:total_count';
@@ -87,20 +83,12 @@ export class EventBuffer extends BaseBuffer {
for (const event of eventsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(event));
if (event.profile_id) {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id
);
}
}
multi.incrby(this.bufferCounterKey, eventsToFlush.length);
await multi.exec();
this.flushRetryCount = 0;
this.pruneHeartbeatMap();
} catch (error) {
// Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -202,58 +190,21 @@ export class EventBuffer extends BaseBuffer {
}
}
public async getBufferSize() {
public getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache();
return await redis.llen(this.queueKey);
});
}
private pruneHeartbeatMap() {
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
for (const [key, ts] of this.lastHeartbeat) {
if (ts < cutoff) {
this.lastHeartbeat.delete(key);
}
}
}
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string
) {
const key = `${projectId}:${profileId}`;
const now = Date.now();
const last = this.lastHeartbeat.get(key) ?? 0;
if (now - last < this.heartbeatRefreshMs) {
return;
}
this.lastHeartbeat.set(key, now);
const zsetKey = `live:visitors:${projectId}`;
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
multi
.zadd(zsetKey, now, profileId)
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
}
public async getActiveVisitorCount(projectId: string): Promise<number> {
const redis = getRedisCache();
const zsetKey = `live:visitors:${projectId}`;
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
const multi = redis.multi();
multi
.zremrangebyscore(zsetKey, '-inf', cutoff)
.zcount(zsetKey, cutoff, '+inf');
const [, count] = (await multi.exec()) as [
[Error | null, any],
[Error | null, number],
];
return count[1] || 0;
const rows = await chQuery<{ count: number }>(
`SELECT uniq(profile_id) AS count
FROM events
WHERE project_id = '${projectId}'
AND profile_id != ''
AND created_at >= now() - INTERVAL 5 MINUTE`
);
return rows[0]?.count ?? 0;
}
}

183
packages/db/src/buffers/profile-buffer.test.ts
View File

@@ -1,6 +1,5 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { getSafeJson } from '@openpanel/json';
import type { IClickhouseProfile } from '../services/profile.service';
// Mock chQuery to avoid hitting real ClickHouse
@@ -36,7 +35,11 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile
}
beforeEach(async () => {
await redis.flushdb();
const keys = [
...await redis.keys('profile*'),
...await redis.keys('lock:profile'),
];
if (keys.length > 0) await redis.del(...keys);
vi.mocked(chQuery).mockResolvedValue([]);
});
@@ -63,64 +66,12 @@ describe('ProfileBuffer', () => {
expect(sizeAfter).toBe(sizeBefore + 1);
});
it('merges subsequent updates via cache (sequential calls)', async () => {
it('concurrent adds: both raw profiles are queued', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Sequential: identify first, then group
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
// Second add should read the cached identify profile and merge groups in
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent identify + group calls preserve all data', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Both calls run concurrently — the per-profile lock serializes them so the
// second one reads the first's result from cache and merges correctly.
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent writes produce one merged buffer entry', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
@@ -128,24 +79,126 @@ describe('ProfileBuffer', () => {
});
const sizeBefore = await profileBuffer.getBufferSize();
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const sizeAfter = await profileBuffer.getBufferSize();
// Both raw profiles are queued; merge happens at flush time
expect(sizeAfter).toBe(sizeBefore + 2);
});
it('merges sequential updates for the same profile at flush time', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('merges concurrent updates for the same profile at flush time', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
await profileBuffer.processBuffer();
const sizeAfter = await profileBuffer.getBufferSize();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
// The second add merges into the first — only 2 buffer entries total
// (one from identify, one merged update with group)
expect(sizeAfter).toBe(sizeBefore + 2);
it('uses existing ClickHouse data for cache misses when merging', async () => {
const existingInClickhouse = makeProfile({
first_name: 'Jane',
email: 'jane@example.com',
groups: ['existing-group'],
});
vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);
// The last entry in the buffer should have both name and group
const rawEntries = await redis.lrange('profile-buffer', 0, -1);
const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e));
const lastEntry = entries[entries.length - 1];
const incomingProfile = makeProfile({
first_name: '',
email: '',
groups: ['new-group'],
});
expect(lastEntry?.first_name).toBe('John');
expect(lastEntry?.groups).toContain('group-abc');
await profileBuffer.add(incomingProfile);
await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('Jane');
expect(cached?.email).toBe('jane@example.com');
expect(cached?.groups).toContain('existing-group');
expect(cached?.groups).toContain('new-group');
});
it('buffer is empty after flush', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
expect(await profileBuffer.getBufferSize()).toBe(1);
await profileBuffer.processBuffer();
expect(await profileBuffer.getBufferSize()).toBe(0);
});
it('retains profiles in queue when ClickHouse insert fails', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
const { ch } = await import('../clickhouse/client');
const insertSpy = vi
.spyOn(ch, 'insert')
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await profileBuffer.processBuffer();
// Profiles must still be in the queue — not lost
expect(await profileBuffer.getBufferSize()).toBe(1);
insertSpy.mockRestore();
});
it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {
vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));
const { ch } = await import('../clickhouse/client');
const insertSpy = vi
.spyOn(ch, 'insert')
.mockResolvedValueOnce(undefined as any);
await profileBuffer.add(makeProfile({ first_name: 'John' }));
await profileBuffer.processBuffer();
// Insert must still have been called — no data loss even when fetch fails
expect(insertSpy).toHaveBeenCalled();
expect(await profileBuffer.getBufferSize()).toBe(0);
insertSpy.mockRestore();
});
});

431
packages/db/src/buffers/profile-buffer.ts
View File

@@ -1,9 +1,6 @@
import { deepMergeObjects } from '@openpanel/common';
import { generateSecureId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json';
import type { ILogger } from '@openpanel/logger';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
@@ -11,29 +8,24 @@ import type { IClickhouseProfile } from '../services/profile.service';
import { BaseBuffer } from './base-buffer';
export class ProfileBuffer extends BaseBuffer {
private batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)
: 200;
private chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
private readonly chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10)
: 1000;
private ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
private readonly ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
? Number.parseInt(process.env.PROFILE_BUFFER_TTL_IN_SECONDS, 10)
: 60 * 60;
/** Max profiles per ClickHouse IN-clause fetch to keep query size bounded */
private readonly fetchChunkSize = process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE, 10)
: 50;
private readonly redisKey = 'profile-buffer';
private readonly redisProfilePrefix = 'profile-cache:';
private redis: Redis;
private releaseLockSha: string | null = null;
private readonly releaseLockScript = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
else
return 0
end
`;
private readonly redis: Redis;
constructor() {
super({
@@ -43,9 +35,6 @@ export class ProfileBuffer extends BaseBuffer {
},
});
this.redis = getRedisCache();
this.redis.script('LOAD', this.releaseLockScript).then((sha) => {
this.releaseLockSha = sha as string;
});
}
private getProfileCacheKey({
@@ -58,243 +47,236 @@ export class ProfileBuffer extends BaseBuffer {
return `${this.redisProfilePrefix}${projectId}:${profileId}`;
}
private async withProfileLock<T>(
public async fetchFromCache(
profileId: string,
projectId: string,
fn: () => Promise<T>
): Promise<T> {
const lockKey = `profile-lock:${projectId}:${profileId}`;
const lockId = generateSecureId('lock');
const maxRetries = 20;
const retryDelayMs = 50;
for (let i = 0; i < maxRetries; i++) {
const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');
if (acquired === 'OK') {
try {
return await fn();
} finally {
if (this.releaseLockSha) {
await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);
} else {
await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);
}
}
}
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
projectId: string
): Promise<IClickhouseProfile | null> {
const cacheKey = this.getProfileCacheKey({ profileId, projectId });
const cached = await this.redis.get(cacheKey);
if (!cached) {
return null;
}
this.logger.error(
'Failed to acquire profile lock, proceeding without lock',
{
profileId,
projectId,
}
);
return fn();
}
async alreadyExists(profile: IClickhouseProfile) {
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
return (await this.redis.exists(cacheKey)) === 1;
return getSafeJson<IClickhouseProfile>(cached);
}
async add(profile: IClickhouseProfile, isFromEvent = false) {
const logger = this.logger.child({
projectId: profile.project_id,
profileId: profile.id,
});
try {
logger.debug('Adding profile');
if (isFromEvent && (await this.alreadyExists(profile))) {
logger.debug('Profile already created, skipping');
return;
}
await this.withProfileLock(profile.id, profile.project_id, async () => {
const existingProfile = await this.fetchProfile(profile, logger);
// Delete any properties that are not server related if we have a non-server profile
if (
existingProfile?.properties.device !== 'server' &&
profile.properties.device === 'server'
) {
profile.properties = omit(
[
'city',
'country',
'region',
'longitude',
'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
profile.properties
);
}
const mergedProfile: IClickhouseProfile = existingProfile
? {
...deepMergeObjects(
existingProfile,
omit(['created_at', 'groups'], profile)
),
groups: uniq([
...(existingProfile.groups ?? []),
...(profile.groups ?? []),
]),
}
: profile;
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
this.logger.debug('Merged profile will be inserted', {
mergedProfile,
existingProfile,
profile,
});
if (isFromEvent) {
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
const result = await this.redis
.multi()
.set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(mergedProfile))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add profile to Redis', {
profile,
cacheKey,
});
const exists = await this.redis.exists(cacheKey);
if (exists === 1) {
return;
}
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
}
this.logger.debug('Current buffer length', {
bufferLength,
batchSize: this.batchSize,
});
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
});
const result = await this.redis
.multi()
.rpush(this.redisKey, JSON.stringify(profile))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add profile to Redis', { profile });
return;
}
const bufferLength = (result?.[2]?.[1] as number) ?? 0;
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
} catch (error) {
this.logger.error('Failed to add profile', { error, profile });
}
}
private async fetchProfile(
profile: IClickhouseProfile,
logger: ILogger
): Promise<IClickhouseProfile | null> {
const existingProfile = await this.fetchFromCache(
profile.id,
profile.project_id
);
if (existingProfile) {
logger.debug('Profile found in Redis');
return existingProfile;
private mergeProfiles(
existing: IClickhouseProfile | null,
incoming: IClickhouseProfile
): IClickhouseProfile {
if (!existing) {
return incoming;
}
return this.fetchFromClickhouse(profile, logger);
}
public async fetchFromCache(
profileId: string,
projectId: string
): Promise<IClickhouseProfile | null> {
const cacheKey = this.getProfileCacheKey({
profileId,
projectId,
});
const existingProfile = await this.redis.get(cacheKey);
if (!existingProfile) {
return null;
let profile = incoming;
if (
existing.properties.device !== 'server' &&
incoming.properties.device === 'server'
) {
profile = {
...incoming,
properties: omit(
[
'city',
'country',
'region',
'longitude',
'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
incoming.properties
),
};
}
return getSafeJson<IClickhouseProfile>(existingProfile);
return {
...deepMergeObjects(existing, omit(['created_at', 'groups'], profile)),
groups: uniq([...(existing.groups ?? []), ...(incoming.groups ?? [])]),
};
}
private async fetchFromClickhouse(
profile: IClickhouseProfile,
logger: ILogger
): Promise<IClickhouseProfile | null> {
logger.debug('Fetching profile from Clickhouse');
const result = await chQuery<IClickhouseProfile>(
`SELECT
id,
project_id,
last_value(nullIf(first_name, '')) as first_name,
last_value(nullIf(last_name, '')) as last_name,
last_value(nullIf(email, '')) as email,
last_value(nullIf(avatar, '')) as avatar,
last_value(is_external) as is_external,
last_value(properties) as properties,
last_value(created_at) as created_at
FROM ${TABLE_NAMES.profiles}
WHERE
id = ${sqlstring.escape(String(profile.id))} AND
project_id = ${sqlstring.escape(profile.project_id)}
${
profile.is_external === false
? ' AND profiles.created_at > now() - INTERVAL 2 DAY'
: ''
private async batchFetchFromClickhouse(
profiles: IClickhouseProfile[]
): Promise<Map<string, IClickhouseProfile>> {
const result = new Map<string, IClickhouseProfile>();
// Non-external (anonymous/device) profiles get a 2-day recency filter to
// avoid pulling stale anonymous sessions from far back.
const external = profiles.filter((p) => p.is_external !== false);
const nonExternal = profiles.filter((p) => p.is_external === false);
const fetchGroup = async (
group: IClickhouseProfile[],
withDateFilter: boolean
) => {
for (const chunk of this.chunks(group, this.fetchChunkSize)) {
const tuples = chunk
.map(
(p) =>
`(${sqlstring.escape(String(p.id))}, ${sqlstring.escape(p.project_id)})`
)
.join(', ');
try {
const rows = await chQuery<IClickhouseProfile>(
`SELECT
id,
project_id,
argMax(nullIf(first_name, ''), ${TABLE_NAMES.profiles}.created_at) as first_name,
argMax(nullIf(last_name, ''), ${TABLE_NAMES.profiles}.created_at) as last_name,
argMax(nullIf(email, ''), ${TABLE_NAMES.profiles}.created_at) as email,
argMax(nullIf(avatar, ''), ${TABLE_NAMES.profiles}.created_at) as avatar,
argMax(is_external, ${TABLE_NAMES.profiles}.created_at) as is_external,
argMax(properties, ${TABLE_NAMES.profiles}.created_at) as properties,
max(created_at) as created_at
FROM ${TABLE_NAMES.profiles}
WHERE (id, project_id) IN (${tuples})
${withDateFilter ? `AND ${TABLE_NAMES.profiles}.created_at > now() - INTERVAL 2 DAY` : ''}
GROUP BY id, project_id`
);
for (const row of rows) {
result.set(`${row.project_id}:${row.id}`, row);
}
} catch (error) {
this.logger.warn(
'Failed to batch fetch profiles from Clickhouse, proceeding without existing data',
{ error, chunkSize: chunk.length }
);
}
GROUP BY id, project_id
ORDER BY created_at DESC
LIMIT 1`
);
logger.debug('Clickhouse fetch result', {
found: !!result[0],
});
return result[0] || null;
}
};
await Promise.all([
fetchGroup(external, false),
fetchGroup(nonExternal, true),
]);
return result;
}
async processBuffer() {
try {
this.logger.debug('Starting profile buffer processing');
const profiles = await this.redis.lrange(
const rawProfiles = await this.redis.lrange(
this.redisKey,
0,
this.batchSize - 1
);
if (profiles.length === 0) {
if (rawProfiles.length === 0) {
this.logger.debug('No profiles to process');
return;
}
this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
const parsedProfiles = profiles.map((p) =>
getSafeJson<IClickhouseProfile>(p)
);
const parsedProfiles = rawProfiles
.map((p) => getSafeJson<IClickhouseProfile>(p))
.filter(Boolean) as IClickhouseProfile[];
for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {
// Merge within batch: collapse multiple updates for the same profile
const mergedInBatch = new Map<string, IClickhouseProfile>();
for (const profile of parsedProfiles) {
const key = `${profile.project_id}:${profile.id}`;
mergedInBatch.set(
key,
this.mergeProfiles(mergedInBatch.get(key) ?? null, profile)
);
}
const uniqueProfiles = Array.from(mergedInBatch.values());
// Check Redis cache for all unique profiles in a single MGET
const cacheKeys = uniqueProfiles.map((p) =>
this.getProfileCacheKey({ profileId: p.id, projectId: p.project_id })
);
const cacheResults = await this.redis.mget(...cacheKeys);
const existingByKey = new Map<string, IClickhouseProfile>();
const cacheMisses: IClickhouseProfile[] = [];
for (let i = 0; i < uniqueProfiles.length; i++) {
const uniqueProfile = uniqueProfiles[i];
if (uniqueProfile) {
const key = `${uniqueProfile.project_id}:${uniqueProfile.id}`;
const cached = cacheResults[i]
? getSafeJson<IClickhouseProfile>(cacheResults[i]!)
: null;
if (cached) {
existingByKey.set(key, cached);
} else {
cacheMisses.push(uniqueProfile);
}
}
}
// Fetch cache misses from ClickHouse in bounded chunks
if (cacheMisses.length > 0) {
const clickhouseResults =
await this.batchFetchFromClickhouse(cacheMisses);
for (const [key, profile] of clickhouseResults) {
existingByKey.set(key, profile);
}
}
// Final merge: in-batch profile + existing (from cache or ClickHouse)
const toInsert: IClickhouseProfile[] = [];
const multi = this.redis.multi();
for (const profile of uniqueProfiles) {
const key = `${profile.project_id}:${profile.id}`;
const merged = this.mergeProfiles(
existingByKey.get(key) ?? null,
profile
);
toInsert.push(merged);
multi.set(
this.getProfileCacheKey({
projectId: profile.project_id,
profileId: profile.id,
}),
JSON.stringify(merged),
'EX',
this.ttlInSeconds
);
}
for (const chunk of this.chunks(toInsert, this.chunkSize)) {
await ch.insert({
table: TABLE_NAMES.profiles,
values: chunk,
@@ -302,22 +284,21 @@ export class ProfileBuffer extends BaseBuffer {
});
}
// Only remove profiles after successful insert and update counter
await this.redis
.multi()
.ltrim(this.redisKey, profiles.length, -1)
.decrby(this.bufferCounterKey, profiles.length)
.exec();
multi
.ltrim(this.redisKey, rawProfiles.length, -1)
.decrby(this.bufferCounterKey, rawProfiles.length);
await multi.exec();
this.logger.debug('Successfully completed profile processing', {
totalProfiles: profiles.length,
totalProfiles: rawProfiles.length,
uniqueProfiles: uniqueProfiles.length,
});
} catch (error) {
this.logger.error('Failed to process buffer', { error });
}
}
async getBufferSize() {
getBufferSize() {
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
}
}

122
packages/db/src/buffers/session-buffer.test.ts Normal file
View File

@@ -0,0 +1,122 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { ch } from '../clickhouse/client';
vi.mock('../clickhouse/client', () => ({
ch: {
insert: vi.fn().mockResolvedValue(undefined),
},
TABLE_NAMES: {
sessions: 'sessions',
},
}));
import { SessionBuffer } from './session-buffer';
import type { IClickhouseEvent } from '../services/event.service';
const redis = getRedisCache();
function makeEvent(overrides: Partial<IClickhouseEvent>): IClickhouseEvent {
return {
id: 'event-1',
project_id: 'project-1',
profile_id: 'profile-1',
device_id: 'device-1',
session_id: 'session-1',
name: 'screen_view',
path: '/home',
origin: '',
referrer: '',
referrer_name: '',
referrer_type: '',
duration: 0,
properties: {},
created_at: new Date().toISOString(),
groups: [],
...overrides,
} as IClickhouseEvent;
}
beforeEach(async () => {
const keys = [
...await redis.keys('session*'),
...await redis.keys('lock:session'),
];
if (keys.length > 0) await redis.del(...keys);
vi.mocked(ch.insert).mockResolvedValue(undefined as any);
});
afterAll(async () => {
try {
await redis.quit();
} catch {}
});
describe('SessionBuffer', () => {
let sessionBuffer: SessionBuffer;
beforeEach(() => {
sessionBuffer = new SessionBuffer();
});
it('adds a new session to the buffer', async () => {
const sizeBefore = await sessionBuffer.getBufferSize();
await sessionBuffer.add(makeEvent({}));
const sizeAfter = await sessionBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore + 1);
});
it('skips session_start and session_end events', async () => {
const sizeBefore = await sessionBuffer.getBufferSize();
await sessionBuffer.add(makeEvent({ name: 'session_start' }));
await sessionBuffer.add(makeEvent({ name: 'session_end' }));
const sizeAfter = await sessionBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore);
});
it('updates existing session on subsequent events', async () => {
const t0 = Date.now();
await sessionBuffer.add(makeEvent({ created_at: new Date(t0).toISOString() }));
// Second event updates the same session — emits old (sign=-1) + new (sign=1)
const sizeBefore = await sessionBuffer.getBufferSize();
await sessionBuffer.add(makeEvent({ created_at: new Date(t0 + 5000).toISOString() }));
const sizeAfter = await sessionBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore + 2);
});
it('processes buffer and inserts sessions into ClickHouse', async () => {
await sessionBuffer.add(makeEvent({}));
const insertSpy = vi
.spyOn(ch, 'insert')
.mockResolvedValueOnce(undefined as any);
await sessionBuffer.processBuffer();
expect(insertSpy).toHaveBeenCalledWith(
expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
);
expect(await sessionBuffer.getBufferSize()).toBe(0);
insertSpy.mockRestore();
});
it('retains sessions in queue when ClickHouse insert fails', async () => {
await sessionBuffer.add(makeEvent({}));
const insertSpy = vi
.spyOn(ch, 'insert')
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await sessionBuffer.processBuffer();
// Sessions must still be in the queue — not lost
expect(await sessionBuffer.getBufferSize()).toBe(1);
insertSpy.mockRestore();
});
});

10
packages/redis/cachable.test.ts
View File

@@ -8,7 +8,10 @@ describe('cachable', () => {
beforeEach(async () => {
redis = getRedisCache();
// Clear any existing cache data for clean tests
const keys = await redis.keys('cachable:*');
const keys = [
...await redis.keys('cachable:*'),
...await redis.keys('test-key*'),
];
if (keys.length > 0) {
await redis.del(...keys);
}
@@ -16,7 +19,10 @@ describe('cachable', () => {
afterEach(async () => {
// Clean up after each test
const keys = await redis.keys('cachable:*');
const keys = [
...await redis.keys('cachable:*'),
...await redis.keys('test-key*'),
];
if (keys.length > 0) {
await redis.del(...keys);
}