fix:buffers

* wip

* remove active visitor counter in redis

* test

* fix profiel query

* fix

apps/api/src/controllers/live.controller.ts

@@ -1,10 +1,7 @@
 import type { WebSocket } from '@fastify/websocket';
 import { eventBuffer } from '@openpanel/db';
 import { setSuperJson } from '@openpanel/json';
-import {
+import { subscribeToPublishedEvent } from '@openpanel/redis';
-  psubscribeToPublishedEvent,
-  subscribeToPublishedEvent,
-} from '@openpanel/redis';
 import { getProjectAccess } from '@openpanel/trpc';
 import { getOrganizationAccess } from '@openpanel/trpc/src/access';
 import type { FastifyRequest } from 'fastify';
@@ -39,19 +36,8 @@ export function wsVisitors(
     }
   );
 
-  const punsubscribe = psubscribeToPublishedEvent(
-    '__keyevent@0__:expired',
-    (key) => {
-      const [, , projectId] = key.split(':');
-      if (projectId === params.projectId) {
-        sendCount();
-      }
-    }
-  );
-
   socket.on('close', () => {
     unsubscribe();
-    punsubscribe();
   });
 }
 

apps/start/src/components/overview/live-counter.tsx

@@ -1,61 +1,25 @@
-import { TooltipComplete } from '@/components/tooltip-complete';
+import { useQueryClient } from '@tanstack/react-query';
-import { useDebounceState } from '@/hooks/use-debounce-state';
+import { useCallback } from 'react';
-import useWS from '@/hooks/use-ws';
-import { useTRPC } from '@/integrations/trpc/react';
-import { cn } from '@/utils/cn';
-import { useQuery, useQueryClient } from '@tanstack/react-query';
-import { useEffect, useRef } from 'react';
 import { toast } from 'sonner';
 import { AnimatedNumber } from '../animated-number';
+import { TooltipComplete } from '@/components/tooltip-complete';
+import { useLiveCounter } from '@/hooks/use-live-counter';
+import { cn } from '@/utils/cn';
 
 export interface LiveCounterProps {
   projectId: string;
   shareId?: string;
 }
 
-const FIFTEEN_SECONDS = 1000 * 30;
-
 export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
-  const trpc = useTRPC();
   const client = useQueryClient();
-  const counter = useDebounceState(0, 1000);
+  const onRefresh = useCallback(() => {
-  const lastRefresh = useRef(Date.now());
+    toast('Refreshed data');
-  const query = useQuery(
+    client.refetchQueries({
-    trpc.overview.liveVisitors.queryOptions({
+      type: 'active',
-      projectId,
+    });
-      shareId,
+  }, [client]);
-    }),
+  const counter = useLiveCounter({ projectId, shareId, onRefresh });
-  );
-
-  useEffect(() => {
-    if (query.data) {
-      counter.set(query.data);
-    }
-  }, [query.data]);
-
-  useWS<number>(
-    `/live/visitors/${projectId}`,
-    (value) => {
-      if (!Number.isNaN(value)) {
-        counter.set(value);
-        if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
-          lastRefresh.current = Date.now();
-          if (!document.hidden) {
-            toast('Refreshed data');
-            client.refetchQueries({
-              type: 'active',
-            });
-          }
-        }
-      }
-    },
-    {
-      debounce: {
-        delay: 1000,
-        maxWait: 5000,
-      },
-    },
-  );
 
   return (
     <TooltipComplete
@@ -66,13 +30,13 @@ export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
           <div
             className={cn(
               'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
-              counter.debounced === 0 && 'bg-destructive opacity-0',
+              counter.debounced === 0 && 'bg-destructive opacity-0'
             )}
           />
           <div
             className={cn(
-              'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
+              'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
-              counter.debounced === 0 && 'bg-destructive',
+              counter.debounced === 0 && 'bg-destructive'
             )}
           />
         </div>

apps/start/src/hooks/use-live-counter.ts

@@ -0,0 +1,81 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query';
+import { useEffect, useRef } from 'react';
+import { useDebounceState } from './use-debounce-state';
+import useWS from './use-ws';
+import { useTRPC } from '@/integrations/trpc/react';
+
+const FIFTEEN_SECONDS = 1000 * 15;
+/** Refetch from API when WS-only updates may be stale (e.g. visitors left). */
+const FALLBACK_STALE_MS = 1000 * 60;
+
+export function useLiveCounter({
+  projectId,
+  shareId,
+  onRefresh,
+}: {
+  projectId: string;
+  shareId?: string;
+  onRefresh?: () => void;
+}) {
+  const trpc = useTRPC();
+  const queryClient = useQueryClient();
+  const counter = useDebounceState(0, 1000);
+  const lastRefresh = useRef(Date.now());
+  const query = useQuery(
+    trpc.overview.liveVisitors.queryOptions({
+      projectId,
+      shareId: shareId ?? undefined,
+    })
+  );
+
+  useEffect(() => {
+    if (query.data) {
+      counter.set(query.data);
+    }
+  }, [query.data]);
+
+  useWS<number>(
+    `/live/visitors/${projectId}`,
+    (value) => {
+      if (!Number.isNaN(value)) {
+        counter.set(value);
+        if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
+          lastRefresh.current = Date.now();
+          if (!document.hidden) {
+            onRefresh?.();
+          }
+        }
+      }
+    },
+    {
+      debounce: {
+        delay: 1000,
+        maxWait: 5000,
+      },
+    }
+  );
+
+  useEffect(() => {
+    const id = setInterval(async () => {
+      if (Date.now() - lastRefresh.current < FALLBACK_STALE_MS) {
+        return;
+      }
+      const data = await queryClient.fetchQuery(
+        trpc.overview.liveVisitors.queryOptions(
+          {
+            projectId,
+            shareId: shareId ?? undefined,
+          },
+          // Default query staleTime is 5m; bypass cache so this reconciliation always hits the API.
+          { staleTime: 0 }
+        )
+      );
+      counter.set(data);
+      lastRefresh.current = Date.now();
+    }, FALLBACK_STALE_MS);
+
+    return () => clearInterval(id);
+  }, [projectId, shareId, trpc, queryClient, counter.set]);
+
+  return counter;
+}

apps/start/src/routes/widget/counter.tsx

@@ -1,12 +1,11 @@
-import { AnimatedNumber } from '@/components/animated-number';
-import { Ping } from '@/components/ping';
-import { useNumber } from '@/hooks/use-numer-formatter';
-import useWS from '@/hooks/use-ws';
-import { useTRPC } from '@/integrations/trpc/react';
-import type { RouterOutputs } from '@/trpc/client';
 import { useQuery, useQueryClient } from '@tanstack/react-query';
 import { createFileRoute } from '@tanstack/react-router';
 import { z } from 'zod';
+import { AnimatedNumber } from '@/components/animated-number';
+import { Ping } from '@/components/ping';
+import useWS from '@/hooks/use-ws';
+import { useTRPC } from '@/integrations/trpc/react';
+import type { RouterOutputs } from '@/trpc/client';
 
 const widgetSearchSchema = z.object({
   shareId: z.string(),
@@ -20,33 +19,33 @@ export const Route = createFileRoute('/widget/counter')({
 });
 
 function RouteComponent() {
-  const { shareId, limit, color } = Route.useSearch();
+  const { shareId } = Route.useSearch();
   const trpc = useTRPC();
 
   // Fetch widget data
   const { data, isLoading } = useQuery(
-    trpc.widget.counter.queryOptions({ shareId }),
+    trpc.widget.counter.queryOptions({ shareId })
   );
 
   if (isLoading) {
     return (
-      <div className="flex items-center gap-2 px-2 h-8">
+      <div className="flex h-8 items-center gap-2 px-2">
         <Ping />
-        <AnimatedNumber value={0} suffix=" unique visitors" />
+        <AnimatedNumber suffix=" unique visitors" value={0} />
       </div>
     );
   }
 
   if (!data) {
     return (
-      <div className="flex items-center gap-2 px-2 h-8">
+      <div className="flex h-8 items-center gap-2 px-2">
         <Ping className="bg-orange-500" />
-        <AnimatedNumber value={0} suffix=" unique visitors" />
+        <AnimatedNumber suffix=" unique visitors" value={0} />
       </div>
     );
   }
 
-  return <CounterWidget shareId={shareId} data={data} />;
+  return <CounterWidget data={data} shareId={shareId} />;
 }
 
 interface RealtimeWidgetProps {
@@ -57,30 +56,29 @@ interface RealtimeWidgetProps {
 function CounterWidget({ shareId, data }: RealtimeWidgetProps) {
   const trpc = useTRPC();
   const queryClient = useQueryClient();
-  const number = useNumber();
 
   // WebSocket subscription for real-time updates
   useWS<number>(
     `/live/visitors/${data.projectId}`,
-    (res) => {
+    () => {
       if (!document.hidden) {
         queryClient.refetchQueries(
-          trpc.widget.counter.queryFilter({ shareId }),
+          trpc.widget.counter.queryFilter({ shareId })
         );
       }
     },
     {
       debounce: {
         delay: 1000,
-        maxWait: 60000,
+        maxWait: 60_000,
       },
-    },
+    }
   );
 
   return (
-    <div className="flex items-center gap-2 px-2 h-8">
+    <div className="flex h-8 items-center gap-2 px-2">
       <Ping />
-      <AnimatedNumber value={data.counter} suffix=" unique visitors" />
+      <AnimatedNumber suffix=" unique visitors" value={data.counter} />
     </div>
   );
 }

apps/start/src/routes/widget/realtime.tsx

@@ -1,3 +1,15 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query';
+import { createFileRoute } from '@tanstack/react-router';
+import type React from 'react';
+import {
+  Bar,
+  BarChart,
+  ResponsiveContainer,
+  Tooltip,
+  XAxis,
+  YAxis,
+} from 'recharts';
+import { z } from 'zod';
 import { AnimatedNumber } from '@/components/animated-number';
 import {
   ChartTooltipContainer,
@@ -14,18 +26,6 @@ import { countries } from '@/translations/countries';
 import type { RouterOutputs } from '@/trpc/client';
 import { cn } from '@/utils/cn';
 import { getChartColor } from '@/utils/theme';
-import { useQuery, useQueryClient } from '@tanstack/react-query';
-import { createFileRoute } from '@tanstack/react-router';
-import type React from 'react';
-import {
-  Bar,
-  BarChart,
-  ResponsiveContainer,
-  Tooltip,
-  XAxis,
-  YAxis,
-} from 'recharts';
-import { z } from 'zod';
 
 const widgetSearchSchema = z.object({
   shareId: z.string(),
@@ -44,7 +44,7 @@ function RouteComponent() {
 
   // Fetch widget data
   const { data: widgetData, isLoading } = useQuery(
-    trpc.widget.realtimeData.queryOptions({ shareId }),
+    trpc.widget.realtimeData.queryOptions({ shareId })
   );
 
   if (isLoading) {
@@ -53,10 +53,10 @@ function RouteComponent() {
 
   if (!widgetData) {
     return (
-      <div className="flex h-screen w-full center-center bg-background text-foreground col p-4">
+      <div className="center-center col flex h-screen w-full bg-background p-4 text-foreground">
-        <LogoSquare className="size-10 mb-4" />
+        <LogoSquare className="mb-4 size-10" />
-        <h1 className="text-xl font-semibold">Widget not found</h1>
+        <h1 className="font-semibold text-xl">Widget not found</h1>
-        <p className="mt-2 text-sm text-muted-foreground">
+        <p className="mt-2 text-muted-foreground text-sm">
           This widget is not available or has been removed.
         </p>
       </div>
@@ -65,10 +65,10 @@ function RouteComponent() {
 
   return (
     <RealtimeWidget
-      shareId={shareId}
-      limit={limit}
-      data={widgetData}
       color={color}
+      data={widgetData}
+      limit={limit}
+      shareId={shareId}
     />
   );
 }
@@ -83,7 +83,6 @@ interface RealtimeWidgetProps {
 function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
   const trpc = useTRPC();
   const queryClient = useQueryClient();
-  const number = useNumber();
 
   // WebSocket subscription for real-time updates
   useWS<number>(
@@ -91,16 +90,16 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
     () => {
       if (!document.hidden) {
         queryClient.refetchQueries(
-          trpc.widget.realtimeData.queryFilter({ shareId }),
+          trpc.widget.realtimeData.queryFilter({ shareId })
         );
       }
     },
     {
       debounce: {
         delay: 1000,
-        maxWait: 60000,
+        maxWait: 60_000,
       },
-    },
+    }
   );
 
   const maxDomain =
@@ -111,8 +110,12 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
     const referrers = data.referrers.length > 0 ? 1 : 0;
     const paths = data.paths.length > 0 ? 1 : 0;
     const value = countries + referrers + paths;
-    if (value === 3) return 'md:grid-cols-3';
+    if (value === 3) {
-    if (value === 2) return 'md:grid-cols-2';
+      return 'md:grid-cols-3';
+    }
+    if (value === 2) {
+      return 'md:grid-cols-2';
+    }
     return 'md:grid-cols-1';
   })();
 
@@ -120,10 +123,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
     <div className="flex h-screen w-full flex-col bg-background text-foreground">
       {/* Header with live counter */}
       <div className="p-6 pb-3">
-        <div className="flex items-center justify-between w-full h-4">
+        <div className="flex h-4 w-full items-center justify-between">
-          <div className="flex items-center gap-3 w-full">
+          <div className="flex w-full items-center gap-3">
             <Ping />
-            <div className="text-sm font-medium text-muted-foreground flex-1">
+            <div className="flex-1 font-medium text-muted-foreground text-sm">
               USERS IN LAST 30 MINUTES
             </div>
             {data.project.domain && <SerieIcon name={data.project.domain} />}
@@ -131,14 +134,14 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
         </div>
 
         <div className="row">
-          <div className="font-mono text-6xl font-bold h-18 text-foreground">
+          <div className="h-18 font-bold font-mono text-6xl text-foreground">
             <AnimatedNumber value={data.liveCount} />
           </div>
         </div>
 
-        <div className="flex h-20 w-full flex-col -mt-4">
+        <div className="-mt-4 flex h-20 w-full flex-col">
           <div className="flex-1">
-            <ResponsiveContainer width="100%" height="100%">
+            <ResponsiveContainer height="100%" width="100%">
               <BarChart
                 data={data.histogram}
                 margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
@@ -148,22 +151,22 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
                   cursor={{ fill: 'var(--def-100)', radius: 4 }}
                 />
                 <XAxis
-                  dataKey="time"
                   axisLine={false}
-                  tickLine={false}
+                  dataKey="time"
+                  interval="preserveStartEnd"
                   tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }}
+                  tickLine={false}
                   ticks={[
                     data.histogram[0].time,
                     data.histogram[data.histogram.length - 1].time,
                   ]}
-                  interval="preserveStartEnd"
                 />
-                <YAxis hide domain={[0, maxDomain]} />
+                <YAxis domain={[0, maxDomain]} hide />
                 <Bar
                   dataKey="sessionCount"
+                  fill={color || 'var(--chart-0)'}
                   isAnimationActive={false}
                   radius={[4, 4, 4, 4]}
-                  fill={color || 'var(--chart-0)'}
                 />
               </BarChart>
             </ResponsiveContainer>
@@ -174,24 +177,24 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
       {(data.countries.length > 0 ||
         data.referrers.length > 0 ||
         data.paths.length > 0) && (
-        <div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar border-t">
+        <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto border-t p-6">
           <div className={cn('grid grid-cols-1 gap-6', grids)}>
             {/* Countries */}
             {data.countries.length > 0 && (
               <div className="flex flex-col">
-                <div className="mb-3 text-xs font-medium text-muted-foreground">
+                <div className="mb-3 font-medium text-muted-foreground text-xs">
                   COUNTRY
                 </div>
                 <div className="col">
                   {(() => {
                     const { visible, rest, restCount } = getRestItems(
                       data.countries,
-                      limit,
+                      limit
                     );
                     return (
                       <>
                         {visible.map((item) => (
-                          <RowItem key={item.country} count={item.count}>
+                          <RowItem count={item.count} key={item.country}>
                             <div className="flex items-center gap-2">
                               <SerieIcon name={item.country} />
                               <span className="text-sm">
@@ -224,19 +227,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
             {/* Referrers */}
             {data.referrers.length > 0 && (
               <div className="flex flex-col">
-                <div className="mb-3 text-xs font-medium text-muted-foreground">
+                <div className="mb-3 font-medium text-muted-foreground text-xs">
                   REFERRER
                 </div>
                 <div className="col">
                   {(() => {
                     const { visible, rest, restCount } = getRestItems(
                       data.referrers,
-                      limit,
+                      limit
                     );
                     return (
                       <>
                         {visible.map((item) => (
-                          <RowItem key={item.referrer} count={item.count}>
+                          <RowItem count={item.count} key={item.referrer}>
                             <div className="flex items-center gap-2">
                               <SerieIcon name={item.referrer} />
                               <span className="truncate text-sm">
@@ -263,19 +266,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
             {/* Paths */}
             {data.paths.length > 0 && (
               <div className="flex flex-col">
-                <div className="mb-3 text-xs font-medium text-muted-foreground">
+                <div className="mb-3 font-medium text-muted-foreground text-xs">
                   PATH
                 </div>
                 <div className="col">
                   {(() => {
                     const { visible, rest, restCount } = getRestItems(
                       data.paths,
-                      limit,
+                      limit
                     );
                     return (
                       <>
                         {visible.map((item) => (
-                          <RowItem key={item.path} count={item.count}>
+                          <RowItem count={item.count} key={item.path}>
                             <span className="truncate text-sm">
                               {item.path}
                             </span>
@@ -303,10 +306,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
 }
 
 // Custom tooltip component that uses portals to escape overflow hidden
-const CustomTooltip = ({ active, payload, coordinate }: any) => {
+const CustomTooltip = ({ active, payload }: any) => {
   const number = useNumber();
 
-  if (!active || !payload || !payload.length) {
+  if (!(active && payload && payload.length)) {
     return null;
   }
 
@@ -328,10 +331,13 @@ const CustomTooltip = ({ active, payload, coordinate }: any) => {
 function RowItem({
   children,
   count,
-}: { children: React.ReactNode; count: number }) {
+}: {
+  children: React.ReactNode;
+  count: number;
+}) {
   const number = useNumber();
   return (
-    <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
+    <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
       {children}
       <span className="font-semibold">{number.short(count)}</span>
     </div>
@@ -340,7 +346,7 @@ function RowItem({
 
 function getRestItems<T extends { count: number }>(
   items: T[],
-  limit: number,
+  limit: number
 ): { visible: T[]; rest: T[]; restCount: number } {
   const visible = items.slice(0, limit);
   const rest = items.slice(limit);
@@ -375,7 +381,7 @@ function RestRow({
           : 'paths';
 
   return (
-    <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
+    <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
       <span className="truncate">
         {firstName} and {otherCount} more {typeLabel}...
       </span>
@@ -434,13 +440,13 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
   const itemCount = Math.min(limit, 5);
 
   return (
-    <div className="flex h-screen w-full flex-col bg-background text-foreground animate-pulse">
+    <div className="flex h-screen w-full animate-pulse flex-col bg-background text-foreground">
       {/* Header with live counter */}
       <div className="border-b p-6 pb-3">
-        <div className="flex items-center justify-between w-full h-4">
+        <div className="flex h-4 w-full items-center justify-between">
-          <div className="flex items-center gap-3 w-full">
+          <div className="flex w-full items-center gap-3">
             <div className="size-2 rounded-full bg-muted" />
-            <div className="text-sm font-medium text-muted-foreground flex-1">
+            <div className="flex-1 font-medium text-muted-foreground text-sm">
               USERS IN LAST 30 MINUTES
             </div>
           </div>
@@ -448,35 +454,35 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
         </div>
 
         <div className="row">
-          <div className="font-mono text-6xl font-bold h-18 flex items-center py-4 gap-1 row">
+          <div className="row flex h-18 items-center gap-1 py-4 font-bold font-mono text-6xl">
-            <div className="h-full w-6 bg-muted rounded" />
+            <div className="h-full w-6 rounded bg-muted" />
-            <div className="h-full w-6 bg-muted rounded" />
+            <div className="h-full w-6 rounded bg-muted" />
           </div>
         </div>
 
-        <div className="flex h-20 w-full flex-col -mt-4 pb-2.5">
+        <div className="-mt-4 flex h-20 w-full flex-col pb-2.5">
-          <div className="flex-1 row gap-1 h-full">
+          <div className="row h-full flex-1 gap-1">
             {SKELETON_HISTOGRAM.map((item, index) => (
               <div
+                className="mt-auto h-full w-full rounded bg-muted"
                 key={index.toString()}
                 style={{ height: `${item}%` }}
-                className="h-full w-full bg-muted rounded mt-auto"
               />
             ))}
           </div>
           <div className="row justify-between pt-2">
-            <div className="h-3 w-8 bg-muted rounded" />
+            <div className="h-3 w-8 rounded bg-muted" />
-            <div className="h-3 w-8 bg-muted rounded" />
+            <div className="h-3 w-8 rounded bg-muted" />
           </div>
         </div>
       </div>
 
-      <div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar">
+      <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto p-6">
         {/* Countries, Referrers, and Paths skeleton */}
         <div className="grid grid-cols-1 gap-6 md:grid-cols-3">
           {/* Countries skeleton */}
           <div className="flex flex-col">
-            <div className="mb-3 text-xs font-medium text-muted-foreground">
+            <div className="mb-3 font-medium text-muted-foreground text-xs">
               COUNTRY
             </div>
             <div className="col">
@@ -488,7 +494,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
 
           {/* Referrers skeleton */}
           <div className="flex flex-col">
-            <div className="mb-3 text-xs font-medium text-muted-foreground">
+            <div className="mb-3 font-medium text-muted-foreground text-xs">
               REFERRER
             </div>
             <div className="col">
@@ -500,7 +506,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
 
           {/* Paths skeleton */}
           <div className="flex flex-col">
-            <div className="mb-3 text-xs font-medium text-muted-foreground">
+            <div className="mb-3 font-medium text-muted-foreground text-xs">
               PATH
             </div>
             <div className="col">
@@ -517,12 +523,12 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
 
 function RowItemSkeleton() {
   return (
-    <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b -mx-3">
+    <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm">
       <div className="flex items-center gap-2">
         <div className="size-5 rounded bg-muted" />
-        <div className="h-4 w-24 bg-muted rounded" />
+        <div className="h-4 w-24 rounded bg-muted" />
       </div>
-      <div className="h-4 w-8 bg-muted rounded" />
+      <div className="h-4 w-8 rounded bg-muted" />
     </div>
   );
 }

packages/db/src/buffers/event-buffer.test.ts

@@ -1,6 +1,7 @@
 import { getRedisCache } from '@openpanel/redis';
 import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import { ch } from '../clickhouse/client';
+import * as chClient from '../clickhouse/client';
+const { ch } = chClient;
 
 // Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
 vi.mock('../services/event.service', () => ({}));
@@ -10,7 +11,8 @@ import { EventBuffer } from './event-buffer';
 const redis = getRedisCache();
 
 beforeEach(async () => {
-  await redis.flushdb();
+  const keys = await redis.keys('event*');
+  if (keys.length > 0) await redis.del(...keys);
 });
 
 afterAll(async () => {
@@ -209,18 +211,16 @@ describe('EventBuffer', () => {
   });
 
   it('tracks active visitors', async () => {
-    const event = {
+    const querySpy = vi
-      project_id: 'p9',
+      .spyOn(chClient, 'chQuery')
-      profile_id: 'u9',
+      .mockResolvedValueOnce([{ count: 2 }] as any);
-      name: 'custom',
-      created_at: new Date().toISOString(),
-    } as any;
-
-    eventBuffer.add(event);
-    await eventBuffer.flush();
 
     const count = await eventBuffer.getActiveVisitorCount('p9');
-    expect(count).toBeGreaterThanOrEqual(1);
+    expect(count).toBe(2);
+    expect(querySpy).toHaveBeenCalledOnce();
+    expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");
+
+    querySpy.mockRestore();
   });
 
   it('handles multiple sessions independently — all events go to buffer', async () => {
@@ -273,4 +273,24 @@ describe('EventBuffer', () => {
 
     expect(await eventBuffer.getBufferSize()).toBe(5);
   });
+
+  it('retains events in queue when ClickHouse insert fails', async () => {
+    eventBuffer.add({
+      project_id: 'p12',
+      name: 'event1',
+      created_at: new Date().toISOString(),
+    } as any);
+    await eventBuffer.flush();
+
+    const insertSpy = vi
+      .spyOn(ch, 'insert')
+      .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+    await eventBuffer.processBuffer();
+
+    // Events must still be in the queue — not lost
+    expect(await eventBuffer.getBufferSize()).toBe(1);
+
+    insertSpy.mockRestore();
+  });
 });

packages/db/src/buffers/event-buffer.ts

@@ -1,6 +1,6 @@
 import { getSafeJson } from '@openpanel/json';
-import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';
+import { getRedisCache, publishEvent } from '@openpanel/redis';
-import { ch } from '../clickhouse/client';
+import { ch, chQuery } from '../clickhouse/client';
 import type { IClickhouseEvent } from '../services/event.service';
 import { BaseBuffer } from './base-buffer';
 
@@ -25,10 +25,6 @@ export class EventBuffer extends BaseBuffer {
   /** Tracks consecutive flush failures for observability; reset on success. */
   private flushRetryCount = 0;
 
-  private activeVisitorsExpiration = 60 * 5; // 5 minutes
-  /** How often (ms) we refresh the heartbeat key + zadd per visitor. */
-  private heartbeatRefreshMs = 60_000; // 1 minute
-  private lastHeartbeat = new Map<string, number>();
   private queueKey = 'event_buffer:queue';
   protected bufferCounterKey = 'event_buffer:total_count';
 
@@ -87,20 +83,12 @@ export class EventBuffer extends BaseBuffer {
 
       for (const event of eventsToFlush) {
         multi.rpush(this.queueKey, JSON.stringify(event));
-        if (event.profile_id) {
-          this.incrementActiveVisitorCount(
-            multi,
-            event.project_id,
-            event.profile_id
-          );
-        }
       }
       multi.incrby(this.bufferCounterKey, eventsToFlush.length);
 
       await multi.exec();
 
       this.flushRetryCount = 0;
-      this.pruneHeartbeatMap();
     } catch (error) {
       // Re-queue failed events at the front to preserve order and avoid data loss
       this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -202,58 +190,21 @@ export class EventBuffer extends BaseBuffer {
     }
   }
 
-  public async getBufferSize() {
+  public getBufferSize() {
     return this.getBufferSizeWithCounter(async () => {
       const redis = getRedisCache();
       return await redis.llen(this.queueKey);
     });
   }
 
-  private pruneHeartbeatMap() {
-    const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
-    for (const [key, ts] of this.lastHeartbeat) {
-      if (ts < cutoff) {
-        this.lastHeartbeat.delete(key);
-      }
-    }
-  }
-
-  private incrementActiveVisitorCount(
-    multi: ReturnType<Redis['multi']>,
-    projectId: string,
-    profileId: string
-  ) {
-    const key = `${projectId}:${profileId}`;
-    const now = Date.now();
-    const last = this.lastHeartbeat.get(key) ?? 0;
-
-    if (now - last < this.heartbeatRefreshMs) {
-      return;
-    }
-
-    this.lastHeartbeat.set(key, now);
-    const zsetKey = `live:visitors:${projectId}`;
-    const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
-    multi
-      .zadd(zsetKey, now, profileId)
-      .set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
-  }
-
   public async getActiveVisitorCount(projectId: string): Promise<number> {
-    const redis = getRedisCache();
+    const rows = await chQuery<{ count: number }>(
-    const zsetKey = `live:visitors:${projectId}`;
+      `SELECT uniq(profile_id) AS count
-    const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
+       FROM events
-
+       WHERE project_id = '${projectId}'
-    const multi = redis.multi();
+         AND profile_id != ''
-    multi
+         AND created_at >= now() - INTERVAL 5 MINUTE`
-      .zremrangebyscore(zsetKey, '-inf', cutoff)
+    );
-      .zcount(zsetKey, cutoff, '+inf');
+    return rows[0]?.count ?? 0;
-
-    const [, count] = (await multi.exec()) as [
-      [Error | null, any],
-      [Error | null, number],
-    ];
-
-    return count[1] || 0;
   }
 }

packages/db/src/buffers/profile-buffer.test.ts

@@ -1,6 +1,5 @@
 import { getRedisCache } from '@openpanel/redis';
 import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import { getSafeJson } from '@openpanel/json';
 import type { IClickhouseProfile } from '../services/profile.service';
 
 // Mock chQuery to avoid hitting real ClickHouse
@@ -36,7 +35,11 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile
 }
 
 beforeEach(async () => {
-  await redis.flushdb();
+  const keys = [
+    ...await redis.keys('profile*'),
+    ...await redis.keys('lock:profile'),
+  ];
+  if (keys.length > 0) await redis.del(...keys);
   vi.mocked(chQuery).mockResolvedValue([]);
 });
 
@@ -63,64 +66,12 @@ describe('ProfileBuffer', () => {
     expect(sizeAfter).toBe(sizeBefore + 1);
   });
 
-  it('merges subsequent updates via cache (sequential calls)', async () => {
+  it('concurrent adds: both raw profiles are queued', async () => {
     const identifyProfile = makeProfile({
       first_name: 'John',
       email: 'john@example.com',
       groups: [],
     });
-
-    const groupProfile = makeProfile({
-      first_name: '',
-      email: '',
-      groups: ['group-abc'],
-    });
-
-    // Sequential: identify first, then group
-    await profileBuffer.add(identifyProfile);
-    await profileBuffer.add(groupProfile);
-
-    // Second add should read the cached identify profile and merge groups in
-    const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
-    expect(cached?.first_name).toBe('John');
-    expect(cached?.email).toBe('john@example.com');
-    expect(cached?.groups).toContain('group-abc');
-  });
-
-  it('race condition: concurrent identify + group calls preserve all data', async () => {
-    const identifyProfile = makeProfile({
-      first_name: 'John',
-      email: 'john@example.com',
-      groups: [],
-    });
-
-    const groupProfile = makeProfile({
-      first_name: '',
-      email: '',
-      groups: ['group-abc'],
-    });
-
-    // Both calls run concurrently — the per-profile lock serializes them so the
-    // second one reads the first's result from cache and merges correctly.
-    await Promise.all([
-      profileBuffer.add(identifyProfile),
-      profileBuffer.add(groupProfile),
-    ]);
-
-    const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
-
-    expect(cached?.first_name).toBe('John');
-    expect(cached?.email).toBe('john@example.com');
-    expect(cached?.groups).toContain('group-abc');
-  });
-
-  it('race condition: concurrent writes produce one merged buffer entry', async () => {
-    const identifyProfile = makeProfile({
-      first_name: 'John',
-      email: 'john@example.com',
-      groups: [],
-    });
-
     const groupProfile = makeProfile({
       first_name: '',
       email: '',
@@ -128,24 +79,126 @@ describe('ProfileBuffer', () => {
     });
 
     const sizeBefore = await profileBuffer.getBufferSize();
+    await Promise.all([
+      profileBuffer.add(identifyProfile),
+      profileBuffer.add(groupProfile),
+    ]);
+    const sizeAfter = await profileBuffer.getBufferSize();
+
+    // Both raw profiles are queued; merge happens at flush time
+    expect(sizeAfter).toBe(sizeBefore + 2);
+  });
+
+  it('merges sequential updates for the same profile at flush time', async () => {
+    const identifyProfile = makeProfile({
+      first_name: 'John',
+      email: 'john@example.com',
+      groups: [],
+    });
+    const groupProfile = makeProfile({
+      first_name: '',
+      email: '',
+      groups: ['group-abc'],
+    });
+
+    await profileBuffer.add(identifyProfile);
+    await profileBuffer.add(groupProfile);
+    await profileBuffer.processBuffer();
+
+    const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
+    expect(cached?.first_name).toBe('John');
+    expect(cached?.email).toBe('john@example.com');
+    expect(cached?.groups).toContain('group-abc');
+  });
+
+  it('merges concurrent updates for the same profile at flush time', async () => {
+    const identifyProfile = makeProfile({
+      first_name: 'John',
+      email: 'john@example.com',
+      groups: [],
+    });
+    const groupProfile = makeProfile({
+      first_name: '',
+      email: '',
+      groups: ['group-abc'],
+    });
 
     await Promise.all([
       profileBuffer.add(identifyProfile),
       profileBuffer.add(groupProfile),
     ]);
+    await profileBuffer.processBuffer();
 
-    const sizeAfter = await profileBuffer.getBufferSize();
+    const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
+    expect(cached?.first_name).toBe('John');
+    expect(cached?.email).toBe('john@example.com');
+    expect(cached?.groups).toContain('group-abc');
+  });
 
-    // The second add merges into the first — only 2 buffer entries total
+  it('uses existing ClickHouse data for cache misses when merging', async () => {
-    // (one from identify, one merged update with group)
+    const existingInClickhouse = makeProfile({
-    expect(sizeAfter).toBe(sizeBefore + 2);
+      first_name: 'Jane',
+      email: 'jane@example.com',
+      groups: ['existing-group'],
+    });
+    vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);
 
-    // The last entry in the buffer should have both name and group
+    const incomingProfile = makeProfile({
-    const rawEntries = await redis.lrange('profile-buffer', 0, -1);
+      first_name: '',
-    const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e));
+      email: '',
-    const lastEntry = entries[entries.length - 1];
+      groups: ['new-group'],
+    });
 
-    expect(lastEntry?.first_name).toBe('John');
+    await profileBuffer.add(incomingProfile);
-    expect(lastEntry?.groups).toContain('group-abc');
+    await profileBuffer.processBuffer();
+
+    const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
+    expect(cached?.first_name).toBe('Jane');
+    expect(cached?.email).toBe('jane@example.com');
+    expect(cached?.groups).toContain('existing-group');
+    expect(cached?.groups).toContain('new-group');
+  });
+
+  it('buffer is empty after flush', async () => {
+    await profileBuffer.add(makeProfile({ first_name: 'John' }));
+    expect(await profileBuffer.getBufferSize()).toBe(1);
+
+    await profileBuffer.processBuffer();
+
+    expect(await profileBuffer.getBufferSize()).toBe(0);
+  });
+
+  it('retains profiles in queue when ClickHouse insert fails', async () => {
+    await profileBuffer.add(makeProfile({ first_name: 'John' }));
+
+    const { ch } = await import('../clickhouse/client');
+    const insertSpy = vi
+      .spyOn(ch, 'insert')
+      .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+    await profileBuffer.processBuffer();
+
+    // Profiles must still be in the queue — not lost
+    expect(await profileBuffer.getBufferSize()).toBe(1);
+
+    insertSpy.mockRestore();
+  });
+
+  it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {
+    vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+    const { ch } = await import('../clickhouse/client');
+    const insertSpy = vi
+      .spyOn(ch, 'insert')
+      .mockResolvedValueOnce(undefined as any);
+
+    await profileBuffer.add(makeProfile({ first_name: 'John' }));
+    await profileBuffer.processBuffer();
+
+    // Insert must still have been called — no data loss even when fetch fails
+    expect(insertSpy).toHaveBeenCalled();
+    expect(await profileBuffer.getBufferSize()).toBe(0);
+
+    insertSpy.mockRestore();
   });
 });

packages/db/src/buffers/profile-buffer.ts

@@ -1,9 +1,6 @@
 import { deepMergeObjects } from '@openpanel/common';
-import { generateSecureId } from '@openpanel/common/server';
 import { getSafeJson } from '@openpanel/json';
-import type { ILogger } from '@openpanel/logger';
 import { getRedisCache, type Redis } from '@openpanel/redis';
-import shallowEqual from 'fast-deep-equal';
 import { omit, uniq } from 'ramda';
 import sqlstring from 'sqlstring';
 import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
@@ -11,29 +8,24 @@ import type { IClickhouseProfile } from '../services/profile.service';
 import { BaseBuffer } from './base-buffer';
 
 export class ProfileBuffer extends BaseBuffer {
-  private batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
+  private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
     ? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)
     : 200;
-  private chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
+  private readonly chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
     ? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10)
     : 1000;
-  private ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
+  private readonly ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
     ? Number.parseInt(process.env.PROFILE_BUFFER_TTL_IN_SECONDS, 10)
     : 60 * 60;
+  /** Max profiles per ClickHouse IN-clause fetch to keep query size bounded */
+  private readonly fetchChunkSize = process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE
+    ? Number.parseInt(process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE, 10)
+    : 50;
 
   private readonly redisKey = 'profile-buffer';
   private readonly redisProfilePrefix = 'profile-cache:';
 
-  private redis: Redis;
+  private readonly redis: Redis;
-  private releaseLockSha: string | null = null;
-
-  private readonly releaseLockScript = `
-    if redis.call("get", KEYS[1]) == ARGV[1] then
-      return redis.call("del", KEYS[1])
-    else
-      return 0
-    end
-  `;
 
   constructor() {
     super({
@@ -43,9 +35,6 @@ export class ProfileBuffer extends BaseBuffer {
       },
     });
     this.redis = getRedisCache();
-    this.redis.script('LOAD', this.releaseLockScript).then((sha) => {
-      this.releaseLockSha = sha as string;
-    });
   }
 
   private getProfileCacheKey({
@@ -58,243 +47,236 @@ export class ProfileBuffer extends BaseBuffer {
     return `${this.redisProfilePrefix}${projectId}:${profileId}`;
   }
 
-  private async withProfileLock<T>(
+  public async fetchFromCache(
     profileId: string,
-    projectId: string,
+    projectId: string
-    fn: () => Promise<T>
+  ): Promise<IClickhouseProfile | null> {
-  ): Promise<T> {
+    const cacheKey = this.getProfileCacheKey({ profileId, projectId });
-    const lockKey = `profile-lock:${projectId}:${profileId}`;
+    const cached = await this.redis.get(cacheKey);
-    const lockId = generateSecureId('lock');
+    if (!cached) {
-    const maxRetries = 20;
+      return null;
-    const retryDelayMs = 50;
-
-    for (let i = 0; i < maxRetries; i++) {
-      const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');
-      if (acquired === 'OK') {
-        try {
-          return await fn();
-        } finally {
-          if (this.releaseLockSha) {
-            await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);
-          } else {
-            await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);
-          }
-        }
-      }
-      await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
     }
-
+    return getSafeJson<IClickhouseProfile>(cached);
-    this.logger.error(
-      'Failed to acquire profile lock, proceeding without lock',
-      {
-        profileId,
-        projectId,
-      }
-    );
-    return fn();
-  }
-
-  async alreadyExists(profile: IClickhouseProfile) {
-    const cacheKey = this.getProfileCacheKey({
-      profileId: profile.id,
-      projectId: profile.project_id,
-    });
-    return (await this.redis.exists(cacheKey)) === 1;
   }
 
   async add(profile: IClickhouseProfile, isFromEvent = false) {
-    const logger = this.logger.child({
-      projectId: profile.project_id,
-      profileId: profile.id,
-    });
-
     try {
-      logger.debug('Adding profile');
+      if (isFromEvent) {
-
-      if (isFromEvent && (await this.alreadyExists(profile))) {
-        logger.debug('Profile already created, skipping');
-        return;
-      }
-
-      await this.withProfileLock(profile.id, profile.project_id, async () => {
-        const existingProfile = await this.fetchProfile(profile, logger);
-
-        // Delete any properties that are not server related if we have a non-server profile
-        if (
-          existingProfile?.properties.device !== 'server' &&
-          profile.properties.device === 'server'
-        ) {
-          profile.properties = omit(
-            [
-              'city',
-              'country',
-              'region',
-              'longitude',
-              'latitude',
-              'os',
-              'osVersion',
-              'browser',
-              'device',
-              'isServer',
-              'os_version',
-              'browser_version',
-            ],
-            profile.properties
-          );
-        }
-
-        const mergedProfile: IClickhouseProfile = existingProfile
-          ? {
-              ...deepMergeObjects(
-                existingProfile,
-                omit(['created_at', 'groups'], profile)
-              ),
-              groups: uniq([
-                ...(existingProfile.groups ?? []),
-                ...(profile.groups ?? []),
-              ]),
-            }
-          : profile;
-
-        if (
-          profile &&
-          existingProfile &&
-          shallowEqual(
-            omit(['created_at'], existingProfile),
-            omit(['created_at'], mergedProfile)
-          )
-        ) {
-          this.logger.debug('Profile not changed, skipping');
-          return;
-        }
-
-        this.logger.debug('Merged profile will be inserted', {
-          mergedProfile,
-          existingProfile,
-          profile,
-        });
-
         const cacheKey = this.getProfileCacheKey({
           profileId: profile.id,
           projectId: profile.project_id,
         });
-
+        const exists = await this.redis.exists(cacheKey);
-        const result = await this.redis
+        if (exists === 1) {
-          .multi()
-          .set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
-          .rpush(this.redisKey, JSON.stringify(mergedProfile))
-          .incr(this.bufferCounterKey)
-          .llen(this.redisKey)
-          .exec();
-
-        if (!result) {
-          this.logger.error('Failed to add profile to Redis', {
-            profile,
-            cacheKey,
-          });
           return;
         }
-        const bufferLength = (result?.[3]?.[1] as number) ?? 0;
+      }
 
-        this.logger.debug('Current buffer length', {
+      const result = await this.redis
-          bufferLength,
+        .multi()
-          batchSize: this.batchSize,
+        .rpush(this.redisKey, JSON.stringify(profile))
-        });
+        .incr(this.bufferCounterKey)
-        if (bufferLength >= this.batchSize) {
+        .llen(this.redisKey)
-          await this.tryFlush();
+        .exec();
-        }
+
-      });
+      if (!result) {
+        this.logger.error('Failed to add profile to Redis', { profile });
+        return;
+      }
+
+      const bufferLength = (result?.[2]?.[1] as number) ?? 0;
+      if (bufferLength >= this.batchSize) {
+        await this.tryFlush();
+      }
     } catch (error) {
       this.logger.error('Failed to add profile', { error, profile });
     }
   }
 
-  private async fetchProfile(
+  private mergeProfiles(
-    profile: IClickhouseProfile,
+    existing: IClickhouseProfile | null,
-    logger: ILogger
+    incoming: IClickhouseProfile
-  ): Promise<IClickhouseProfile | null> {
+  ): IClickhouseProfile {
-    const existingProfile = await this.fetchFromCache(
+    if (!existing) {
-      profile.id,
+      return incoming;
-      profile.project_id
-    );
-    if (existingProfile) {
-      logger.debug('Profile found in Redis');
-      return existingProfile;
     }
 
-    return this.fetchFromClickhouse(profile, logger);
+    let profile = incoming;
-  }
+    if (
-
+      existing.properties.device !== 'server' &&
-  public async fetchFromCache(
+      incoming.properties.device === 'server'
-    profileId: string,
+    ) {
-    projectId: string
+      profile = {
-  ): Promise<IClickhouseProfile | null> {
+        ...incoming,
-    const cacheKey = this.getProfileCacheKey({
+        properties: omit(
-      profileId,
+          [
-      projectId,
+            'city',
-    });
+            'country',
-    const existingProfile = await this.redis.get(cacheKey);
+            'region',
-    if (!existingProfile) {
+            'longitude',
-      return null;
+            'latitude',
+            'os',
+            'osVersion',
+            'browser',
+            'device',
+            'isServer',
+            'os_version',
+            'browser_version',
+          ],
+          incoming.properties
+        ),
+      };
     }
-    return getSafeJson<IClickhouseProfile>(existingProfile);
+
+    return {
+      ...deepMergeObjects(existing, omit(['created_at', 'groups'], profile)),
+      groups: uniq([...(existing.groups ?? []), ...(incoming.groups ?? [])]),
+    };
   }
 
-  private async fetchFromClickhouse(
+  private async batchFetchFromClickhouse(
-    profile: IClickhouseProfile,
+    profiles: IClickhouseProfile[]
-    logger: ILogger
+  ): Promise<Map<string, IClickhouseProfile>> {
-  ): Promise<IClickhouseProfile | null> {
+    const result = new Map<string, IClickhouseProfile>();
-    logger.debug('Fetching profile from Clickhouse');
+
-    const result = await chQuery<IClickhouseProfile>(
+    // Non-external (anonymous/device) profiles get a 2-day recency filter to
-      `SELECT 
+    // avoid pulling stale anonymous sessions from far back.
-        id, 
+    const external = profiles.filter((p) => p.is_external !== false);
-        project_id,
+    const nonExternal = profiles.filter((p) => p.is_external === false);
-        last_value(nullIf(first_name, '')) as first_name, 
+
-        last_value(nullIf(last_name, '')) as last_name, 
+    const fetchGroup = async (
-        last_value(nullIf(email, '')) as email, 
+      group: IClickhouseProfile[],
-        last_value(nullIf(avatar, '')) as avatar, 
+      withDateFilter: boolean
-        last_value(is_external) as is_external, 
+    ) => {
-        last_value(properties) as properties, 
+      for (const chunk of this.chunks(group, this.fetchChunkSize)) {
-        last_value(created_at) as created_at
+        const tuples = chunk
-      FROM ${TABLE_NAMES.profiles} 
+          .map(
-      WHERE 
+            (p) =>
-        id = ${sqlstring.escape(String(profile.id))} AND 
+              `(${sqlstring.escape(String(p.id))}, ${sqlstring.escape(p.project_id)})`
-        project_id = ${sqlstring.escape(profile.project_id)}
+          )
-        ${
+          .join(', ');
-          profile.is_external === false
+        try {
-            ? ' AND profiles.created_at > now() - INTERVAL 2 DAY'
+          const rows = await chQuery<IClickhouseProfile>(
-            : ''
+            `SELECT
+              id,
+              project_id,
+              argMax(nullIf(first_name, ''), ${TABLE_NAMES.profiles}.created_at) as first_name,
+              argMax(nullIf(last_name, ''), ${TABLE_NAMES.profiles}.created_at) as last_name,
+              argMax(nullIf(email, ''), ${TABLE_NAMES.profiles}.created_at) as email,
+              argMax(nullIf(avatar, ''), ${TABLE_NAMES.profiles}.created_at) as avatar,
+              argMax(is_external, ${TABLE_NAMES.profiles}.created_at) as is_external,
+              argMax(properties, ${TABLE_NAMES.profiles}.created_at) as properties,
+              max(created_at) as created_at
+            FROM ${TABLE_NAMES.profiles}
+            WHERE (id, project_id) IN (${tuples})
+            ${withDateFilter ? `AND ${TABLE_NAMES.profiles}.created_at > now() - INTERVAL 2 DAY` : ''}
+            GROUP BY id, project_id`
+          );
+          for (const row of rows) {
+            result.set(`${row.project_id}:${row.id}`, row);
+          }
+        } catch (error) {
+          this.logger.warn(
+            'Failed to batch fetch profiles from Clickhouse, proceeding without existing data',
+            { error, chunkSize: chunk.length }
+          );
         }
-      GROUP BY id, project_id 
+      }
-      ORDER BY created_at DESC 
+    };
-      LIMIT 1`
+
-    );
+    await Promise.all([
-    logger.debug('Clickhouse fetch result', {
+      fetchGroup(external, false),
-      found: !!result[0],
+      fetchGroup(nonExternal, true),
-    });
+    ]);
-    return result[0] || null;
+
+    return result;
   }
 
   async processBuffer() {
     try {
       this.logger.debug('Starting profile buffer processing');
-      const profiles = await this.redis.lrange(
+      const rawProfiles = await this.redis.lrange(
         this.redisKey,
         0,
         this.batchSize - 1
       );
 
-      if (profiles.length === 0) {
+      if (rawProfiles.length === 0) {
         this.logger.debug('No profiles to process');
         return;
       }
 
-      this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
+      const parsedProfiles = rawProfiles
-      const parsedProfiles = profiles.map((p) =>
+        .map((p) => getSafeJson<IClickhouseProfile>(p))
-        getSafeJson<IClickhouseProfile>(p)
+        .filter(Boolean) as IClickhouseProfile[];
-      );
 
-      for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {
+      // Merge within batch: collapse multiple updates for the same profile
+      const mergedInBatch = new Map<string, IClickhouseProfile>();
+      for (const profile of parsedProfiles) {
+        const key = `${profile.project_id}:${profile.id}`;
+        mergedInBatch.set(
+          key,
+          this.mergeProfiles(mergedInBatch.get(key) ?? null, profile)
+        );
+      }
+
+      const uniqueProfiles = Array.from(mergedInBatch.values());
+
+      // Check Redis cache for all unique profiles in a single MGET
+      const cacheKeys = uniqueProfiles.map((p) =>
+        this.getProfileCacheKey({ profileId: p.id, projectId: p.project_id })
+      );
+      const cacheResults = await this.redis.mget(...cacheKeys);
+
+      const existingByKey = new Map<string, IClickhouseProfile>();
+      const cacheMisses: IClickhouseProfile[] = [];
+      for (let i = 0; i < uniqueProfiles.length; i++) {
+        const uniqueProfile = uniqueProfiles[i];
+        if (uniqueProfile) {
+          const key = `${uniqueProfile.project_id}:${uniqueProfile.id}`;
+          const cached = cacheResults[i]
+            ? getSafeJson<IClickhouseProfile>(cacheResults[i]!)
+            : null;
+          if (cached) {
+            existingByKey.set(key, cached);
+          } else {
+            cacheMisses.push(uniqueProfile);
+          }
+        }
+      }
+
+      // Fetch cache misses from ClickHouse in bounded chunks
+      if (cacheMisses.length > 0) {
+        const clickhouseResults =
+          await this.batchFetchFromClickhouse(cacheMisses);
+        for (const [key, profile] of clickhouseResults) {
+          existingByKey.set(key, profile);
+        }
+      }
+
+      // Final merge: in-batch profile + existing (from cache or ClickHouse)
+      const toInsert: IClickhouseProfile[] = [];
+      const multi = this.redis.multi();
+
+      for (const profile of uniqueProfiles) {
+        const key = `${profile.project_id}:${profile.id}`;
+        const merged = this.mergeProfiles(
+          existingByKey.get(key) ?? null,
+          profile
+        );
+        toInsert.push(merged);
+        multi.set(
+          this.getProfileCacheKey({
+            projectId: profile.project_id,
+            profileId: profile.id,
+          }),
+          JSON.stringify(merged),
+          'EX',
+          this.ttlInSeconds
+        );
+      }
+
+      for (const chunk of this.chunks(toInsert, this.chunkSize)) {
         await ch.insert({
           table: TABLE_NAMES.profiles,
           values: chunk,
@@ -302,22 +284,21 @@ export class ProfileBuffer extends BaseBuffer {
         });
       }
 
-      // Only remove profiles after successful insert and update counter
+      multi
-      await this.redis
+        .ltrim(this.redisKey, rawProfiles.length, -1)
-        .multi()
+        .decrby(this.bufferCounterKey, rawProfiles.length);
-        .ltrim(this.redisKey, profiles.length, -1)
+      await multi.exec();
-        .decrby(this.bufferCounterKey, profiles.length)
-        .exec();
 
       this.logger.debug('Successfully completed profile processing', {
-        totalProfiles: profiles.length,
+        totalProfiles: rawProfiles.length,
+        uniqueProfiles: uniqueProfiles.length,
       });
     } catch (error) {
       this.logger.error('Failed to process buffer', { error });
     }
   }
 
-  async getBufferSize() {
+  getBufferSize() {
     return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
   }
 }

packages/db/src/buffers/session-buffer.test.ts

@@ -0,0 +1,122 @@
+import { getRedisCache } from '@openpanel/redis';
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { ch } from '../clickhouse/client';
+
+vi.mock('../clickhouse/client', () => ({
+  ch: {
+    insert: vi.fn().mockResolvedValue(undefined),
+  },
+  TABLE_NAMES: {
+    sessions: 'sessions',
+  },
+}));
+
+import { SessionBuffer } from './session-buffer';
+import type { IClickhouseEvent } from '../services/event.service';
+
+const redis = getRedisCache();
+
+function makeEvent(overrides: Partial<IClickhouseEvent>): IClickhouseEvent {
+  return {
+    id: 'event-1',
+    project_id: 'project-1',
+    profile_id: 'profile-1',
+    device_id: 'device-1',
+    session_id: 'session-1',
+    name: 'screen_view',
+    path: '/home',
+    origin: '',
+    referrer: '',
+    referrer_name: '',
+    referrer_type: '',
+    duration: 0,
+    properties: {},
+    created_at: new Date().toISOString(),
+    groups: [],
+    ...overrides,
+  } as IClickhouseEvent;
+}
+
+beforeEach(async () => {
+  const keys = [
+    ...await redis.keys('session*'),
+    ...await redis.keys('lock:session'),
+  ];
+  if (keys.length > 0) await redis.del(...keys);
+  vi.mocked(ch.insert).mockResolvedValue(undefined as any);
+});
+
+afterAll(async () => {
+  try {
+    await redis.quit();
+  } catch {}
+});
+
+describe('SessionBuffer', () => {
+  let sessionBuffer: SessionBuffer;
+
+  beforeEach(() => {
+    sessionBuffer = new SessionBuffer();
+  });
+
+  it('adds a new session to the buffer', async () => {
+    const sizeBefore = await sessionBuffer.getBufferSize();
+    await sessionBuffer.add(makeEvent({}));
+    const sizeAfter = await sessionBuffer.getBufferSize();
+
+    expect(sizeAfter).toBe(sizeBefore + 1);
+  });
+
+  it('skips session_start and session_end events', async () => {
+    const sizeBefore = await sessionBuffer.getBufferSize();
+    await sessionBuffer.add(makeEvent({ name: 'session_start' }));
+    await sessionBuffer.add(makeEvent({ name: 'session_end' }));
+    const sizeAfter = await sessionBuffer.getBufferSize();
+
+    expect(sizeAfter).toBe(sizeBefore);
+  });
+
+  it('updates existing session on subsequent events', async () => {
+    const t0 = Date.now();
+    await sessionBuffer.add(makeEvent({ created_at: new Date(t0).toISOString() }));
+
+    // Second event updates the same session — emits old (sign=-1) + new (sign=1)
+    const sizeBefore = await sessionBuffer.getBufferSize();
+    await sessionBuffer.add(makeEvent({ created_at: new Date(t0 + 5000).toISOString() }));
+    const sizeAfter = await sessionBuffer.getBufferSize();
+
+    expect(sizeAfter).toBe(sizeBefore + 2);
+  });
+
+  it('processes buffer and inserts sessions into ClickHouse', async () => {
+    await sessionBuffer.add(makeEvent({}));
+
+    const insertSpy = vi
+      .spyOn(ch, 'insert')
+      .mockResolvedValueOnce(undefined as any);
+
+    await sessionBuffer.processBuffer();
+
+    expect(insertSpy).toHaveBeenCalledWith(
+      expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
+    );
+    expect(await sessionBuffer.getBufferSize()).toBe(0);
+
+    insertSpy.mockRestore();
+  });
+
+  it('retains sessions in queue when ClickHouse insert fails', async () => {
+    await sessionBuffer.add(makeEvent({}));
+
+    const insertSpy = vi
+      .spyOn(ch, 'insert')
+      .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+    await sessionBuffer.processBuffer();
+
+    // Sessions must still be in the queue — not lost
+    expect(await sessionBuffer.getBufferSize()).toBe(1);
+
+    insertSpy.mockRestore();
+  });
+});

packages/redis/cachable.test.ts

@@ -8,7 +8,10 @@ describe('cachable', () => {
   beforeEach(async () => {
     redis = getRedisCache();
     // Clear any existing cache data for clean tests
-    const keys = await redis.keys('cachable:*');
+    const keys = [
+      ...await redis.keys('cachable:*'),
+      ...await redis.keys('test-key*'),
+    ];
     if (keys.length > 0) {
       await redis.del(...keys);
     }
@@ -16,7 +19,10 @@ describe('cachable', () => {
 
   afterEach(async () => {
     // Clean up after each test
-    const keys = await redis.keys('cachable:*');
+    const keys = [
+      ...await redis.keys('cachable:*'),
+      ...await redis.keys('test-key*'),
+    ];
     if (keys.length > 0) {
       await redis.del(...keys);
     }