zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

remove active visitor counter in redis

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-24 12:18:17 +01:00
parent 8dd3966f31
commit 52be76cc26
7 changed files with 215 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
apps/api/src/controllers/live.controller.ts
View File

@@ -1,10 +1,7 @@
import type { WebSocket } from '@fastify/websocket';
import { eventBuffer } from '@openpanel/db';
import { setSuperJson } from '@openpanel/json';
import {
psubscribeToPublishedEvent,
subscribeToPublishedEvent,
} from '@openpanel/redis';
import { subscribeToPublishedEvent } from '@openpanel/redis';
import { getProjectAccess } from '@openpanel/trpc';
import { getOrganizationAccess } from '@openpanel/trpc/src/access';
import type { FastifyRequest } from 'fastify';
@@ -39,19 +36,8 @@ export function wsVisitors(
}
);
const punsubscribe = psubscribeToPublishedEvent(
'__keyevent@0__:expired',
(key) => {
const [, , projectId] = key.split(':');
if (projectId === params.projectId) {
sendCount();
}
}
);
socket.on('close', () => {
unsubscribe();
punsubscribe();
});
}

66
apps/start/src/components/overview/live-counter.tsx
View File

@@ -1,61 +1,25 @@
import { TooltipComplete } from '@/components/tooltip-complete';
import { useDebounceState } from '@/hooks/use-debounce-state';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import { cn } from '@/utils/cn';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef } from 'react';
import { useQueryClient } from '@tanstack/react-query';
import { useCallback } from 'react';
import { toast } from 'sonner';
import { AnimatedNumber } from '../animated-number';
import { TooltipComplete } from '@/components/tooltip-complete';
import { useLiveCounter } from '@/hooks/use-live-counter';
import { cn } from '@/utils/cn';
export interface LiveCounterProps {
projectId: string;
shareId?: string;
}
const FIFTEEN_SECONDS = 1000 * 30;
export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
const trpc = useTRPC();
const client = useQueryClient();
const counter = useDebounceState(0, 1000);
const lastRefresh = useRef(Date.now());
const query = useQuery(
trpc.overview.liveVisitors.queryOptions({
projectId,
shareId,
}),
);
useEffect(() => {
if (query.data) {
counter.set(query.data);
}
}, [query.data]);
useWS<number>(
`/live/visitors/${projectId}`,
(value) => {
if (!Number.isNaN(value)) {
counter.set(value);
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
lastRefresh.current = Date.now();
if (!document.hidden) {
toast('Refreshed data');
client.refetchQueries({
type: 'active',
});
}
}
}
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
},
);
const onRefresh = useCallback(() => {
toast('Refreshed data');
client.refetchQueries({
type: 'active',
});
}, [client]);
const counter = useLiveCounter({ projectId, shareId, onRefresh });
return (
<TooltipComplete
@@ -66,13 +30,13 @@ export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
<div
className={cn(
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
counter.debounced === 0 && 'bg-destructive opacity-0',
counter.debounced === 0 && 'bg-destructive opacity-0'
)}
/>
<div
className={cn(
'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
counter.debounced === 0 && 'bg-destructive',
'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
counter.debounced === 0 && 'bg-destructive'
)}
/>
</div>

81
apps/start/src/hooks/use-live-counter.ts Normal file
View File

@@ -0,0 +1,81 @@
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef } from 'react';
import { useDebounceState } from './use-debounce-state';
import useWS from './use-ws';
import { useTRPC } from '@/integrations/trpc/react';
const FIFTEEN_SECONDS = 1000 * 15;
/** Refetch from API when WS-only updates may be stale (e.g. visitors left). */
const FALLBACK_STALE_MS = 1000 * 60;
export function useLiveCounter({
projectId,
shareId,
onRefresh,
}: {
projectId: string;
shareId?: string;
onRefresh?: () => void;
}) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const counter = useDebounceState(0, 1000);
const lastRefresh = useRef(Date.now());
const query = useQuery(
trpc.overview.liveVisitors.queryOptions({
projectId,
shareId: shareId ?? undefined,
})
);
useEffect(() => {
if (query.data) {
counter.set(query.data);
}
}, [query.data]);
useWS<number>(
`/live/visitors/${projectId}`,
(value) => {
if (!Number.isNaN(value)) {
counter.set(value);
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
lastRefresh.current = Date.now();
if (!document.hidden) {
onRefresh?.();
}
}
}
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
}
);
useEffect(() => {
const id = setInterval(async () => {
if (Date.now() - lastRefresh.current < FALLBACK_STALE_MS) {
return;
}
const data = await queryClient.fetchQuery(
trpc.overview.liveVisitors.queryOptions(
{
projectId,
shareId: shareId ?? undefined,
},
// Default query staleTime is 5m; bypass cache so this reconciliation always hits the API.
{ staleTime: 0 }
)
);
counter.set(data);
lastRefresh.current = Date.now();
}, FALLBACK_STALE_MS);
return () => clearInterval(id);
}, [projectId, shareId, trpc, queryClient, counter.set]);
return counter;
}

38
apps/start/src/routes/widget/counter.tsx
View File

@@ -1,12 +1,11 @@
import { AnimatedNumber } from '@/components/animated-number';
import { Ping } from '@/components/ping';
import { useNumber } from '@/hooks/use-numer-formatter';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import type { RouterOutputs } from '@/trpc/client';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number';
import { Ping } from '@/components/ping';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import type { RouterOutputs } from '@/trpc/client';
const widgetSearchSchema = z.object({
shareId: z.string(),
@@ -20,33 +19,33 @@ export const Route = createFileRoute('/widget/counter')({
});
function RouteComponent() {
const { shareId, limit, color } = Route.useSearch();
const { shareId } = Route.useSearch();
const trpc = useTRPC();
// Fetch widget data
const { data, isLoading } = useQuery(
trpc.widget.counter.queryOptions({ shareId }),
trpc.widget.counter.queryOptions({ shareId })
);
if (isLoading) {
return (
<div className="flex items-center gap-2 px-2 h-8">
<div className="flex h-8 items-center gap-2 px-2">
<Ping />
<AnimatedNumber value={0} suffix=" unique visitors" />
<AnimatedNumber suffix=" unique visitors" value={0} />
</div>
);
}
if (!data) {
return (
<div className="flex items-center gap-2 px-2 h-8">
<div className="flex h-8 items-center gap-2 px-2">
<Ping className="bg-orange-500" />
<AnimatedNumber value={0} suffix=" unique visitors" />
<AnimatedNumber suffix=" unique visitors" value={0} />
</div>
);
}
return <CounterWidget shareId={shareId} data={data} />;
return <CounterWidget data={data} shareId={shareId} />;
}
interface RealtimeWidgetProps {
@@ -57,30 +56,29 @@ interface RealtimeWidgetProps {
function CounterWidget({ shareId, data }: RealtimeWidgetProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const number = useNumber();
// WebSocket subscription for real-time updates
useWS<number>(
`/live/visitors/${data.projectId}`,
(res) => {
() => {
if (!document.hidden) {
queryClient.refetchQueries(
trpc.widget.counter.queryFilter({ shareId }),
trpc.widget.counter.queryFilter({ shareId })
);
}
},
{
debounce: {
delay: 1000,
maxWait: 60000,
maxWait: 60_000,
},
},
}
);
return (
<div className="flex items-center gap-2 px-2 h-8">
<div className="flex h-8 items-center gap-2 px-2">
<Ping />
<AnimatedNumber value={data.counter} suffix=" unique visitors" />
<AnimatedNumber suffix=" unique visitors" value={data.counter} />
</div>
);
}

150
apps/start/src/routes/widget/realtime.tsx
View File

@@ -1,3 +1,15 @@
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import type React from 'react';
import {
Bar,
BarChart,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number';
import {
ChartTooltipContainer,
@@ -14,18 +26,6 @@ import { countries } from '@/translations/countries';
import type { RouterOutputs } from '@/trpc/client';
import { cn } from '@/utils/cn';
import { getChartColor } from '@/utils/theme';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import type React from 'react';
import {
Bar,
BarChart,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
import { z } from 'zod';
const widgetSearchSchema = z.object({
shareId: z.string(),
@@ -44,7 +44,7 @@ function RouteComponent() {
// Fetch widget data
const { data: widgetData, isLoading } = useQuery(
trpc.widget.realtimeData.queryOptions({ shareId }),
trpc.widget.realtimeData.queryOptions({ shareId })
);
if (isLoading) {
@@ -53,10 +53,10 @@ function RouteComponent() {
if (!widgetData) {
return (
<div className="flex h-screen w-full center-center bg-background text-foreground col p-4">
<LogoSquare className="size-10 mb-4" />
<h1 className="text-xl font-semibold">Widget not found</h1>
<p className="mt-2 text-sm text-muted-foreground">
<div className="center-center col flex h-screen w-full bg-background p-4 text-foreground">
<LogoSquare className="mb-4 size-10" />
<h1 className="font-semibold text-xl">Widget not found</h1>
<p className="mt-2 text-muted-foreground text-sm">
This widget is not available or has been removed.
</p>
</div>
@@ -65,10 +65,10 @@ function RouteComponent() {
return (
<RealtimeWidget
shareId={shareId}
limit={limit}
data={widgetData}
color={color}
data={widgetData}
limit={limit}
shareId={shareId}
/>
);
}
@@ -83,7 +83,6 @@ interface RealtimeWidgetProps {
function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const number = useNumber();
// WebSocket subscription for real-time updates
useWS<number>(
@@ -91,16 +90,16 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
() => {
if (!document.hidden) {
queryClient.refetchQueries(
trpc.widget.realtimeData.queryFilter({ shareId }),
trpc.widget.realtimeData.queryFilter({ shareId })
);
}
},
{
debounce: {
delay: 1000,
maxWait: 60000,
maxWait: 60_000,
},
},
}
);
const maxDomain =
@@ -111,8 +110,12 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const referrers = data.referrers.length > 0 ? 1 : 0;
const paths = data.paths.length > 0 ? 1 : 0;
const value = countries + referrers + paths;
if (value === 3) return 'md:grid-cols-3';
if (value === 2) return 'md:grid-cols-2';
if (value === 3) {
return 'md:grid-cols-3';
}
if (value === 2) {
return 'md:grid-cols-2';
}
return 'md:grid-cols-1';
})();
@@ -120,10 +123,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
<div className="flex h-screen w-full flex-col bg-background text-foreground">
{/* Header with live counter */}
<div className="p-6 pb-3">
<div className="flex items-center justify-between w-full h-4">
<div className="flex items-center gap-3 w-full">
<div className="flex h-4 w-full items-center justify-between">
<div className="flex w-full items-center gap-3">
<Ping />
<div className="text-sm font-medium text-muted-foreground flex-1">
<div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES
</div>
{data.project.domain && <SerieIcon name={data.project.domain} />}
@@ -131,14 +134,14 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
</div>
<div className="row">
<div className="font-mono text-6xl font-bold h-18 text-foreground">
<div className="h-18 font-bold font-mono text-6xl text-foreground">
<AnimatedNumber value={data.liveCount} />
</div>
</div>
<div className="flex h-20 w-full flex-col -mt-4">
<div className="-mt-4 flex h-20 w-full flex-col">
<div className="flex-1">
<ResponsiveContainer width="100%" height="100%">
<ResponsiveContainer height="100%" width="100%">
<BarChart
data={data.histogram}
margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
@@ -148,22 +151,22 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
cursor={{ fill: 'var(--def-100)', radius: 4 }}
/>
<XAxis
dataKey="time"
axisLine={false}
tickLine={false}
dataKey="time"
interval="preserveStartEnd"
tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }}
tickLine={false}
ticks={[
data.histogram[0].time,
data.histogram[data.histogram.length - 1].time,
]}
interval="preserveStartEnd"
/>
<YAxis hide domain={[0, maxDomain]} />
<YAxis domain={[0, maxDomain]} hide />
<Bar
dataKey="sessionCount"
fill={color || 'var(--chart-0)'}
isAnimationActive={false}
radius={[4, 4, 4, 4]}
fill={color || 'var(--chart-0)'}
/>
</BarChart>
</ResponsiveContainer>
@@ -174,24 +177,24 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{(data.countries.length > 0 ||
data.referrers.length > 0 ||
data.paths.length > 0) && (
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar border-t">
<div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto border-t p-6">
<div className={cn('grid grid-cols-1 gap-6', grids)}>
{/* Countries */}
{data.countries.length > 0 && (
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.countries,
limit,
limit
);
return (
<>
{visible.map((item) => (
<RowItem key={item.country} count={item.count}>
<RowItem count={item.count} key={item.country}>
<div className="flex items-center gap-2">
<SerieIcon name={item.country} />
<span className="text-sm">
@@ -224,19 +227,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Referrers */}
{data.referrers.length > 0 && (
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.referrers,
limit,
limit
);
return (
<>
{visible.map((item) => (
<RowItem key={item.referrer} count={item.count}>
<RowItem count={item.count} key={item.referrer}>
<div className="flex items-center gap-2">
<SerieIcon name={item.referrer} />
<span className="truncate text-sm">
@@ -263,19 +266,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Paths */}
{data.paths.length > 0 && (
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
PATH
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.paths,
limit,
limit
);
return (
<>
{visible.map((item) => (
<RowItem key={item.path} count={item.count}>
<RowItem count={item.count} key={item.path}>
<span className="truncate text-sm">
{item.path}
</span>
@@ -303,10 +306,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
}
// Custom tooltip component that uses portals to escape overflow hidden
const CustomTooltip = ({ active, payload, coordinate }: any) => {
const CustomTooltip = ({ active, payload }: any) => {
const number = useNumber();
if (!active || !payload || !payload.length) {
if (!(active && payload && payload.length)) {
return null;
}
@@ -328,10 +331,13 @@ const CustomTooltip = ({ active, payload, coordinate }: any) => {
function RowItem({
children,
count,
}: { children: React.ReactNode; count: number }) {
}: {
children: React.ReactNode;
count: number;
}) {
const number = useNumber();
return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
{children}
<span className="font-semibold">{number.short(count)}</span>
</div>
@@ -340,7 +346,7 @@ function RowItem({
function getRestItems<T extends { count: number }>(
items: T[],
limit: number,
limit: number
): { visible: T[]; rest: T[]; restCount: number } {
const visible = items.slice(0, limit);
const rest = items.slice(limit);
@@ -375,7 +381,7 @@ function RestRow({
: 'paths';
return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
<span className="truncate">
{firstName} and {otherCount} more {typeLabel}...
</span>
@@ -434,13 +440,13 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
const itemCount = Math.min(limit, 5);
return (
<div className="flex h-screen w-full flex-col bg-background text-foreground animate-pulse">
<div className="flex h-screen w-full animate-pulse flex-col bg-background text-foreground">
{/* Header with live counter */}
<div className="border-b p-6 pb-3">
<div className="flex items-center justify-between w-full h-4">
<div className="flex items-center gap-3 w-full">
<div className="flex h-4 w-full items-center justify-between">
<div className="flex w-full items-center gap-3">
<div className="size-2 rounded-full bg-muted" />
<div className="text-sm font-medium text-muted-foreground flex-1">
<div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES
</div>
</div>
@@ -448,35 +454,35 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
</div>
<div className="row">
<div className="font-mono text-6xl font-bold h-18 flex items-center py-4 gap-1 row">
<div className="h-full w-6 bg-muted rounded" />
<div className="h-full w-6 bg-muted rounded" />
<div className="row flex h-18 items-center gap-1 py-4 font-bold font-mono text-6xl">
<div className="h-full w-6 rounded bg-muted" />
<div className="h-full w-6 rounded bg-muted" />
</div>
</div>
<div className="flex h-20 w-full flex-col -mt-4 pb-2.5">
<div className="flex-1 row gap-1 h-full">
<div className="-mt-4 flex h-20 w-full flex-col pb-2.5">
<div className="row h-full flex-1 gap-1">
{SKELETON_HISTOGRAM.map((item, index) => (
<div
className="mt-auto h-full w-full rounded bg-muted"
key={index.toString()}
style={{ height: `${item}%` }}
className="h-full w-full bg-muted rounded mt-auto"
/>
))}
</div>
<div className="row justify-between pt-2">
<div className="h-3 w-8 bg-muted rounded" />
<div className="h-3 w-8 bg-muted rounded" />
<div className="h-3 w-8 rounded bg-muted" />
<div className="h-3 w-8 rounded bg-muted" />
</div>
</div>
</div>
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar">
<div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto p-6">
{/* Countries, Referrers, and Paths skeleton */}
<div className="grid grid-cols-1 gap-6 md:grid-cols-3">
{/* Countries skeleton */}
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY
</div>
<div className="col">
@@ -488,7 +494,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Referrers skeleton */}
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER
</div>
<div className="col">
@@ -500,7 +506,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Paths skeleton */}
<div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground">
<div className="mb-3 font-medium text-muted-foreground text-xs">
PATH
</div>
<div className="col">
@@ -517,12 +523,12 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
function RowItemSkeleton() {
return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b -mx-3">
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm">
<div className="flex items-center gap-2">
<div className="size-5 rounded bg-muted" />
<div className="h-4 w-24 bg-muted rounded" />
<div className="h-4 w-24 rounded bg-muted" />
</div>
<div className="h-4 w-8 bg-muted rounded" />
<div className="h-4 w-8 rounded bg-muted" />
</div>
);
}

26
packages/db/src/buffers/event-buffer.test.ts
View File

@@ -1,6 +1,7 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { ch } from '../clickhouse/client';
import * as chClient from '../clickhouse/client';
const { ch } = chClient;
// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
vi.mock('../services/event.service', () => ({}));
@@ -10,10 +11,7 @@ import { EventBuffer } from './event-buffer';
const redis = getRedisCache();
beforeEach(async () => {
const keys = [
...await redis.keys('event*'),
...await redis.keys('live:*'),
];
const keys = await redis.keys('event*');
if (keys.length > 0) await redis.del(...keys);
});
@@ -213,18 +211,16 @@ describe('EventBuffer', () => {
});
it('tracks active visitors', async () => {
const event = {
project_id: 'p9',
profile_id: 'u9',
name: 'custom',
created_at: new Date().toISOString(),
} as any;
eventBuffer.add(event);
await eventBuffer.flush();
const querySpy = vi
.spyOn(chClient, 'chQuery')
.mockResolvedValueOnce([{ count: 2 }] as any);
const count = await eventBuffer.getActiveVisitorCount('p9');
expect(count).toBeGreaterThanOrEqual(1);
expect(count).toBe(2);
expect(querySpy).toHaveBeenCalledOnce();
expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");
querySpy.mockRestore();
});
it('handles multiple sessions independently — all events go to buffer', async () => {

71
packages/db/src/buffers/event-buffer.ts
View File

@@ -1,6 +1,6 @@
import { getSafeJson } from '@openpanel/json';
import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import { getRedisCache, publishEvent } from '@openpanel/redis';
import { ch, chQuery } from '../clickhouse/client';
import type { IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
@@ -25,10 +25,6 @@ export class EventBuffer extends BaseBuffer {
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
private activeVisitorsExpiration = 60 * 5; // 5 minutes
/** How often (ms) we refresh the heartbeat key + zadd per visitor. */
private heartbeatRefreshMs = 60_000; // 1 minute
private lastHeartbeat = new Map<string, number>();
private queueKey = 'event_buffer:queue';
protected bufferCounterKey = 'event_buffer:total_count';
@@ -87,20 +83,12 @@ export class EventBuffer extends BaseBuffer {
for (const event of eventsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(event));
if (event.profile_id) {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id
);
}
}
multi.incrby(this.bufferCounterKey, eventsToFlush.length);
await multi.exec();
this.flushRetryCount = 0;
this.pruneHeartbeatMap();
} catch (error) {
// Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -202,58 +190,21 @@ export class EventBuffer extends BaseBuffer {
}
}
public async getBufferSize() {
public getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache();
return await redis.llen(this.queueKey);
});
}
private pruneHeartbeatMap() {
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
for (const [key, ts] of this.lastHeartbeat) {
if (ts < cutoff) {
this.lastHeartbeat.delete(key);
}
}
}
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string
) {
const key = `${projectId}:${profileId}`;
const now = Date.now();
const last = this.lastHeartbeat.get(key) ?? 0;
if (now - last < this.heartbeatRefreshMs) {
return;
}
this.lastHeartbeat.set(key, now);
const zsetKey = `live:visitors:${projectId}`;
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
multi
.zadd(zsetKey, now, profileId)
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
}
public async getActiveVisitorCount(projectId: string): Promise<number> {
const redis = getRedisCache();
const zsetKey = `live:visitors:${projectId}`;
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
const multi = redis.multi();
multi
.zremrangebyscore(zsetKey, '-inf', cutoff)
.zcount(zsetKey, cutoff, '+inf');
const [, count] = (await multi.exec()) as [
[Error | null, any],
[Error | null, number],
];
return count[1] || 0;
const rows = await chQuery<{ count: number }>(
`SELECT uniq(profile_id) AS count
FROM events
WHERE project_id = '${projectId}'
AND profile_id != ''
AND created_at >= now() - INTERVAL 5 MINUTE`
);
return rows[0]?.count ?? 0;
}
}