wip

apps/api/src/controllers/live.controller.ts

@@ -1,16 +1,13 @@
-import type { FastifyRequest } from 'fastify';
-import superjson from 'superjson';
-
 import type { WebSocket } from '@fastify/websocket';
-import {
+import { eventBuffer } from '@openpanel/db';
-  eventBuffer,
-  getProfileById,
-  transformMinimalEvent,
-} from '@openpanel/db';
 import { setSuperJson } from '@openpanel/json';
-import { subscribeToPublishedEvent } from '@openpanel/redis';
+import {
+  psubscribeToPublishedEvent,
+  subscribeToPublishedEvent,
+} from '@openpanel/redis';
 import { getProjectAccess } from '@openpanel/trpc';
 import { getOrganizationAccess } from '@openpanel/trpc/src/access';
+import type { FastifyRequest } from 'fastify';
 
 export function wsVisitors(
   socket: WebSocket,
@@ -18,19 +15,38 @@ export function wsVisitors(
     Params: {
       projectId: string;
     };
-  }>,
+  }>
 ) {
   const { params } = req;
-  const unsubscribe = subscribeToPublishedEvent('events', 'saved', (event) => {
+  const sendCount = () => {
-    if (event?.projectId === params.projectId) {
+    eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
-      eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
+      socket.send(String(count));
-        socket.send(String(count));
+    });
-      });
+  };
+
+  const unsubscribe = subscribeToPublishedEvent(
+    'events',
+    'batch',
+    ({ projectId }) => {
+      if (projectId === params.projectId) {
+        sendCount();
+      }
     }
-  });
+  );
+
+  const punsubscribe = psubscribeToPublishedEvent(
+    '__keyevent@0__:expired',
+    (key) => {
+      const [, , projectId] = key.split(':');
+      if (projectId === params.projectId) {
+        sendCount();
+      }
+    }
+  );
 
   socket.on('close', () => {
     unsubscribe();
+    punsubscribe();
   });
 }
 
@@ -42,18 +58,10 @@ export async function wsProjectEvents(
     };
     Querystring: {
       token?: string;
-      type?: 'saved' | 'received';
     };
-  }>,
+  }>
 ) {
-  const { params, query } = req;
+  const { params } = req;
-  const type = query.type || 'saved';
-
-  if (!['saved', 'received'].includes(type)) {
-    socket.send('Invalid type');
-    socket.close();
-    return;
-  }
 
   const userId = req.session?.userId;
   if (!userId) {
@@ -67,24 +75,20 @@ export async function wsProjectEvents(
     projectId: params.projectId,
   });
 
+  if (!access) {
+    socket.send('No access');
+    socket.close();
+    return;
+  }
+
   const unsubscribe = subscribeToPublishedEvent(
     'events',
-    type,
+    'batch',
-    async (event) => {
+    ({ projectId, count }) => {
-      if (event.projectId === params.projectId) {
+      if (projectId === params.projectId) {
-        const profile = await getProfileById(event.profileId, event.projectId);
+        socket.send(setSuperJson({ count }));
-        socket.send(
-          superjson.stringify(
-            access
-              ? {
-                  ...event,
-                  profile,
-                }
-              : transformMinimalEvent(event),
-          ),
-        );
       }
-    },
+    }
   );
 
   socket.on('close', () => unsubscribe());
@@ -96,7 +100,7 @@ export async function wsProjectNotifications(
     Params: {
       projectId: string;
     };
-  }>,
+  }>
 ) {
   const { params } = req;
   const userId = req.session?.userId;
@@ -123,9 +127,9 @@ export async function wsProjectNotifications(
     'created',
     (notification) => {
       if (notification.projectId === params.projectId) {
-        socket.send(superjson.stringify(notification));
+        socket.send(setSuperJson(notification));
       }
-    },
+    }
   );
 
   socket.on('close', () => unsubscribe());
@@ -137,7 +141,7 @@ export async function wsOrganizationEvents(
     Params: {
       organizationId: string;
     };
-  }>,
+  }>
 ) {
   const { params } = req;
   const userId = req.session?.userId;
@@ -164,7 +168,7 @@ export async function wsOrganizationEvents(
     'subscription_updated',
     (message) => {
       socket.send(setSuperJson(message));
-    },
+    }
   );
 
   socket.on('close', () => unsubscribe());

apps/start/src/components/events/event-listener.tsx

@@ -1,3 +1,4 @@
+import { AnimatedNumber } from '../animated-number';
 import {
   Tooltip,
   TooltipContent,
@@ -8,71 +9,53 @@ import { useDebounceState } from '@/hooks/use-debounce-state';
 import useWS from '@/hooks/use-ws';
 import { cn } from '@/utils/cn';
 
-import type { IServiceEvent, IServiceEventMinimal } from '@openpanel/db';
-import { useParams } from '@tanstack/react-router';
-import { AnimatedNumber } from '../animated-number';
-
 export default function EventListener({
   onRefresh,
 }: {
   onRefresh: () => void;
 }) {
-  const params = useParams({
-    strict: false,
-  });
   const { projectId } = useAppParams();
   const counter = useDebounceState(0, 1000);
-  useWS<IServiceEventMinimal | IServiceEvent>(
+  useWS<{ count: number }>(
     `/live/events/${projectId}`,
-    (event) => {
+    ({ count }) => {
-      if (event) {
+      counter.set((prev) => prev + count);
-        const isProfilePage = !!params?.profileId;
-        if (isProfilePage) {
-          const profile = 'profile' in event ? event.profile : null;
-          if (profile?.id === params?.profileId) {
-            counter.set((prev) => prev + 1);
-          }
-          return;
-        }
-
-        counter.set((prev) => prev + 1);
-      }
     },
     {
       debounce: {
         delay: 1000,
         maxWait: 5000,
       },
-    },
+    }
   );
 
   return (
     <Tooltip>
       <TooltipTrigger asChild>
         <button
-          type="button"
+          className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
           onClick={() => {
             counter.set(0);
             onRefresh();
           }}
-          className="flex h-8 items-center gap-2 rounded-md border border-border bg-card px-3 font-medium leading-none"
+          type="button"
         >
           <div className="relative">
             <div
               className={cn(
-                'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
+                'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all'
               )}
             />
             <div
               className={cn(
-                'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
+                'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all'
               )}
             />
           </div>
           {counter.debounced === 0 ? (
             'Listening'
           ) : (
-            <AnimatedNumber value={counter.debounced} suffix=" new events" />
+            <AnimatedNumber suffix=" new events" value={counter.debounced} />
           )}
         </button>
       </TooltipTrigger>

apps/start/src/components/events/table/index.tsx

@@ -35,6 +35,7 @@ type Props = {
     >,
     unknown
   >;
+  showEventListener?: boolean;
 };
 
 const LOADING_DATA = [{}, {}, {}, {}, {}, {}, {}, {}, {}] as IServiceEvent[];
@@ -215,7 +216,7 @@ const VirtualizedEventsTable = ({
   );
 };
 
-export const EventsTable = ({ query }: Props) => {
+export const EventsTable = ({ query, showEventListener = false }: Props) => {
   const { isLoading } = query;
   const columns = useColumns();
 
@@ -272,7 +273,7 @@ export const EventsTable = ({ query }: Props) => {
 
   return (
     <>
-      <EventsTableToolbar query={query} table={table} />
+      <EventsTableToolbar query={query} table={table} showEventListener={showEventListener} />
       <VirtualizedEventsTable table={table} data={data} isLoading={isLoading} />
       <div className="w-full h-10 center-center pt-4" ref={inViewportRef}>
         <div
@@ -291,9 +292,11 @@ export const EventsTable = ({ query }: Props) => {
 function EventsTableToolbar({
   query,
   table,
+  showEventListener,
 }: {
   query: Props['query'];
   table: Table<IServiceEvent>;
+  showEventListener: boolean;
 }) {
   const { projectId } = useAppParams();
   const [startDate, setStartDate] = useQueryState(
@@ -305,7 +308,7 @@ function EventsTableToolbar({
   return (
     <DataTableToolbarContainer>
       <div className="flex flex-1 flex-wrap items-center gap-2">
-        <EventListener onRefresh={() => query.refetch()} />
+        {showEventListener && <EventListener onRefresh={() => query.refetch()} />}
         <Button
           variant="outline"
           size="sm"

apps/start/src/components/onboarding/onboarding-verify-listener.tsx

@@ -1,31 +1,13 @@
-import type {
+import type { IServiceEvent } from '@openpanel/db';
-  IServiceClient,
-  IServiceEvent,
-  IServiceProject,
-} from '@openpanel/db';
 import { CheckCircle2Icon, CheckIcon, Loader2 } from 'lucide-react';
-import { useState } from 'react';
-import useWS from '@/hooks/use-ws';
 import { cn } from '@/utils/cn';
 import { timeAgo } from '@/utils/date';
 
 interface Props {
-  project: IServiceProject;
-  client: IServiceClient | null;
   events: IServiceEvent[];
-  onVerified: (verified: boolean) => void;
 }
 
-const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
+const VerifyListener = ({ events }: Props) => {
-  const [events, setEvents] = useState<IServiceEvent[]>(_events ?? []);
-  useWS<IServiceEvent>(
-    `/live/events/${client?.projectId}?type=received`,
-    (data) => {
-      setEvents((prev) => [...prev, data]);
-      onVerified(true);
-    }
-  );
-
   const isConnected = events.length > 0;
 
   const renderIcon = () => {
@@ -49,16 +31,18 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
       <div
         className={cn(
           'flex gap-6 rounded-xl p-4 md:p-6',
-          isConnected ? 'bg-emerald-100 dark:bg-emerald-700' : 'bg-blue-500/10'
+          isConnected
+            ? 'bg-emerald-100 dark:bg-emerald-700/10'
+            : 'bg-blue-500/10'
         )}
       >
         {renderIcon()}
         <div className="flex-1">
           <div className="font-semibold text-foreground/90 text-lg leading-normal">
-            {isConnected ? 'Success' : 'Waiting for events'}
+            {isConnected ? 'Successfully connected' : 'Waiting for events'}
           </div>
           {isConnected ? (
-            <div className="flex flex-col-reverse">
+            <div className="mt-2 flex flex-col-reverse gap-1">
               {events.length > 5 && (
                 <div className="flex items-center gap-2">
                   <CheckIcon size={14} />{' '}
@@ -69,7 +53,7 @@ const VerifyListener = ({ client, events: _events, onVerified }: Props) => {
                 <div className="flex items-center gap-2" key={event.id}>
                   <CheckIcon size={14} />{' '}
                   <span className="font-medium">{event.name}</span>{' '}
-                  <span className="ml-auto text-emerald-800">
+                  <span className="ml-auto text-foreground/50 text-sm">
                     {timeAgo(event.createdAt, 'round')}
                   </span>
                 </div>

apps/start/src/components/realtime/realtime-active-sessions.tsx

@@ -1,11 +1,9 @@
-import { useTRPC } from '@/integrations/trpc/react';
 import { useQuery } from '@tanstack/react-query';
 import { AnimatePresence, motion } from 'framer-motion';
-import { useEffect, useState } from 'react';
+import { ProjectLink } from '../links';
-
+import { SerieIcon } from '../report-chart/common/serie-icon';
-import useWS from '@/hooks/use-ws';
+import { useTRPC } from '@/integrations/trpc/react';
-import type { IServiceEvent } from '@openpanel/db';
+import { formatTimeAgoOrDateTime } from '@/utils/date';
-import { EventItem } from '../events/table/item';
 
 interface RealtimeActiveSessionsProps {
   projectId: string;
@@ -17,64 +15,52 @@ export function RealtimeActiveSessions({
   limit = 10,
 }: RealtimeActiveSessionsProps) {
   const trpc = useTRPC();
-  const activeSessionsQuery = useQuery(
+  const { data: sessions = [] } = useQuery(
-    trpc.realtime.activeSessions.queryOptions({
+    trpc.realtime.activeSessions.queryOptions(
-      projectId,
+      { projectId },
-    }),
+      { refetchInterval: 5000 }
+    )
   );
 
-  const [state, setState] = useState<IServiceEvent[]>([]);
-
-  // Update state when initial data loads
-  useEffect(() => {
-    if (activeSessionsQuery.data && state.length === 0) {
-      setState(activeSessionsQuery.data);
-    }
-  }, [activeSessionsQuery.data, state]);
-
-  // Set up WebSocket connection for real-time updates
-  useWS<IServiceEvent>(
-    `/live/events/${projectId}`,
-    (session) => {
-      setState((prev) => {
-        // Add new session and remove duplicates, keeping most recent
-        const filtered = prev.filter((s) => s.id !== session.id);
-        return [session, ...filtered].slice(0, limit);
-      });
-    },
-    {
-      debounce: {
-        delay: 1000,
-        maxWait: 5000,
-      },
-    },
-  );
-
-  const sessions = state.length > 0 ? state : (activeSessionsQuery.data ?? []);
-
   return (
-    <div className="col h-full max-md:hidden">
+    <div className="col card h-full max-md:hidden">
-      <div className="hide-scrollbar h-full overflow-y-auto pb-10">
+      <div className="hide-scrollbar h-full overflow-y-auto">
-        <AnimatePresence mode="popLayout" initial={false}>
+        <AnimatePresence initial={false} mode="popLayout">
-          <div className="col gap-4">
+          <div className="col divide-y">
-            {sessions.map((session) => (
+            {sessions.slice(0, limit).map((session) => (
               <motion.div
-                key={session.id}
-                layout
-                // initial={{ opacity: 0, x: -200, scale: 0.8 }}
                 animate={{ opacity: 1, x: 0, scale: 1 }}
                 exit={{ opacity: 0, x: 200, scale: 0.8 }}
+                key={session.id}
+                layout
                 transition={{ duration: 0.4, type: 'spring', stiffness: 300 }}
               >
-                <EventItem
+                <ProjectLink
-                  event={session}
+                  className="relative block p-4 py-3 pr-14"
-                  viewOptions={{
+                  href={`/sessions/${session.sessionId}`}
-                    properties: false,
+                >
-                    origin: false,
+                  <div className="col flex-1 gap-1">
-                    queryString: false,
+                    {session.name === 'screen_view' && (
-                  }}
+                      <span className="text-muted-foreground text-xs leading-normal/80">
-                  className="w-full"
+                        {session.origin}
-                />
+                      </span>
+                    )}
+                    <span className="font-medium text-sm leading-normal">
+                      {session.name === 'screen_view'
+                        ? session.path
+                        : session.name}
+                    </span>
+                    <span className="text-muted-foreground text-xs">
+                      {formatTimeAgoOrDateTime(session.createdAt)}
+                    </span>
+                  </div>
+                  <div className="row absolute top-1/2 right-4 origin-right -translate-y-1/2 scale-50 gap-2">
+                    <SerieIcon name={session.referrerName} />
+                    <SerieIcon name={session.os} />
+                    <SerieIcon name={session.browser} />
+                    <SerieIcon name={session.device} />
+                  </div>
+                </ProjectLink>
               </motion.div>
             ))}
           </div>

apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx

@@ -42,5 +42,5 @@ function Component() {
     ),
   );
 
-  return <EventsTable query={query} />;
+  return <EventsTable query={query} showEventListener />;
 }

apps/start/src/routes/_app.$organizationId.$projectId.realtime.tsx

@@ -1,3 +1,5 @@
+import { keepPreviousData, useQuery } from '@tanstack/react-query';
+import { createFileRoute } from '@tanstack/react-router';
 import { Fullscreen, FullscreenClose } from '@/components/fullscreen-toggle';
 import RealtimeMap from '@/components/realtime/map';
 import { RealtimeActiveSessions } from '@/components/realtime/realtime-active-sessions';
@@ -7,12 +9,10 @@ import { RealtimePaths } from '@/components/realtime/realtime-paths';
 import { RealtimeReferrals } from '@/components/realtime/realtime-referrals';
 import RealtimeReloader from '@/components/realtime/realtime-reloader';
 import { useTRPC } from '@/integrations/trpc/react';
-import { PAGE_TITLES, createProjectTitle } from '@/utils/title';
+import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
-import { keepPreviousData, useQuery } from '@tanstack/react-query';
-import { createFileRoute } from '@tanstack/react-router';
 
 export const Route = createFileRoute(
-  '/_app/$organizationId/$projectId/realtime',
+  '/_app/$organizationId/$projectId/realtime'
 )({
   component: Component,
   head: () => {
@@ -36,8 +36,8 @@ function Component() {
       },
       {
         placeholderData: keepPreviousData,
-      },
+      }
-    ),
+    )
   );
 
   return (
@@ -47,7 +47,7 @@ function Component() {
         <RealtimeReloader projectId={projectId} />
 
         <div className="row relative">
-          <div className="overflow-hidden aspect-[4/2] w-full">
+          <div className="aspect-[4/2] w-full overflow-hidden">
             <RealtimeMap
               markers={coordinatesQuery.data ?? []}
               sidebarConfig={{
@@ -56,18 +56,17 @@ function Component() {
               }}
             />
           </div>
-          <div className="absolute top-8 left-8 bottom-0 col gap-4">
+          <div className="col absolute top-8 bottom-4 left-8 gap-4">
-            <div className="card p-4 w-72 bg-background/90">
+            <div className="card w-72 bg-background/90 p-4">
               <RealtimeLiveHistogram projectId={projectId} />
             </div>
-            <div className="w-72 flex-1 min-h-0 relative">
+            <div className="relative min-h-0 w-72 flex-1">
               <RealtimeActiveSessions projectId={projectId} />
-              <div className="absolute bottom-0 left-0 right-0 h-10 bg-gradient-to-t from-def-100 to-transparent" />
             </div>
           </div>
         </div>
 
-        <div className="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4 p-4 pt-4 md:p-8 md:pt-0">
+        <div className="grid grid-cols-1 gap-4 p-4 pt-4 md:grid-cols-2 md:p-8 md:pt-0 xl:grid-cols-3">
           <div>
             <RealtimeGeo projectId={projectId} />
           </div>

apps/start/src/routes/_steps.onboarding.$projectId.verify.tsx

@@ -1,7 +1,6 @@
 import { useQuery } from '@tanstack/react-query';
 import { createFileRoute, Link, redirect } from '@tanstack/react-router';
 import { BoxSelectIcon } from 'lucide-react';
-import { useEffect, useState } from 'react';
 import { ButtonContainer } from '@/components/button-container';
 import { FullPageEmptyState } from '@/components/full-page-empty-state';
 import FullPageLoadingState from '@/components/full-page-loading-state';
@@ -33,22 +32,21 @@ export const Route = createFileRoute('/_steps/onboarding/$projectId/verify')({
 });
 
 function Component() {
-  const [isVerified, setIsVerified] = useState(false);
   const { projectId } = Route.useParams();
   const trpc = useTRPC();
-  const { data: events, refetch } = useQuery(
+  const { data: events } = useQuery(
-    trpc.event.events.queryOptions({ projectId })
+    trpc.event.events.queryOptions(
+      { projectId },
+      {
+        refetchInterval: 2500,
+      }
+    )
   );
+  const isVerified = events?.data && events.data.length > 0;
   const { data: project } = useQuery(
     trpc.project.getProjectWithClients.queryOptions({ projectId })
   );
 
-  useEffect(() => {
-    if (events && events.data.length > 0) {
-      setIsVerified(true);
-    }
-  }, [events]);
-
   if (!project) {
     return (
       <FullPageEmptyState icon={BoxSelectIcon} title="Project not found" />
@@ -64,15 +62,7 @@ function Component() {
     <div className="flex min-h-0 flex-1 flex-col">
       <div className="scrollbar-thin flex-1 overflow-y-auto">
         <div className="col gap-8 p-4">
-          <VerifyListener
+          <VerifyListener events={events?.data ?? []} />
-            client={client}
-            events={events?.data ?? []}
-            onVerified={() => {
-              refetch();
-              setIsVerified(true);
-            }}
-            project={project}
-          />
 
           <VerifyFaq project={project} />
         </div>

packages/db/src/buffers/event-buffer.ts

@@ -2,7 +2,6 @@ import { getSafeJson } from '@openpanel/json';
 import {
   type Redis,
   getRedisCache,
-  getRedisPub,
   publishEvent,
 } from '@openpanel/redis';
 import { ch } from '../clickhouse/client';
@@ -53,14 +52,10 @@ export class EventBuffer extends BaseBuffer {
   /** Tracks consecutive flush failures for observability; reset on success. */
   private flushRetryCount = 0;
 
-  private publishThrottleMs = process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS
-    ? Number.parseInt(process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS, 10)
-    : 1000;
-  private lastPublishTime = 0;
-  private pendingPublishEvent: IClickhouseEvent | null = null;
-  private publishTimer: ReturnType<typeof setTimeout> | null = null;
-
   private activeVisitorsExpiration = 60 * 5; // 5 minutes
+  /** How often (ms) we refresh the heartbeat key + zadd per visitor. */
+  private heartbeatRefreshMs = 60_000; // 1 minute
+  private lastHeartbeat = new Map<string, number>();
   private queueKey = 'event_buffer:queue';
   protected bufferCounterKey = 'event_buffer:total_count';
 
@@ -194,7 +189,7 @@ return added
     }
   }
 
-  add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
+  add(event: IClickhouseEvent) {
     const eventJson = JSON.stringify(event);
 
     let type: PendingEvent['type'] = 'regular';
@@ -218,11 +213,6 @@ return added
       type,
     };
 
-    if (_multi) {
-      this.addToMulti(_multi, pendingEvent);
-      return;
-    }
-
     this.pendingEvents.push(pendingEvent);
 
     if (this.pendingEvents.length >= this.microBatchMaxSize) {
@@ -318,11 +308,7 @@ return added
       await multi.exec();
 
       this.flushRetryCount = 0;
-
+      this.pruneHeartbeatMap();
-      const lastEvent = eventsToFlush[eventsToFlush.length - 1];
-      if (lastEvent) {
-        this.scheduleThrottledPublish(lastEvent.event);
-      }
     } catch (error) {
       // Re-queue failed events at the front to preserve order and avoid data loss
       this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -335,41 +321,13 @@ return added
       });
     } finally {
       this.isFlushing = false;
-    }
+      // Events may have accumulated while we were flushing; schedule another flush if needed
-  }
+      if (this.pendingEvents.length > 0 && !this.flushTimer) {
-
+        this.flushTimer = setTimeout(() => {
-  private scheduleThrottledPublish(event: IClickhouseEvent) {
+          this.flushTimer = null;
-    this.pendingPublishEvent = event;
+          this.flushLocalBuffer();
-
+        }, this.microBatchIntervalMs);
-    const now = Date.now();
+      }
-    const timeSinceLastPublish = now - this.lastPublishTime;
-
-    if (timeSinceLastPublish >= this.publishThrottleMs) {
-      this.executeThrottledPublish();
-      return;
-    }
-
-    if (!this.publishTimer) {
-      const delay = this.publishThrottleMs - timeSinceLastPublish;
-      this.publishTimer = setTimeout(() => {
-        this.publishTimer = null;
-        this.executeThrottledPublish();
-      }, delay);
-    }
-  }
-
-  private executeThrottledPublish() {
-    if (!this.pendingPublishEvent) {
-      return;
-    }
-
-    const event = this.pendingPublishEvent;
-    this.pendingPublishEvent = null;
-    this.lastPublishTime = Date.now();
-
-    const result = publishEvent('events', 'received', transformEvent(event));
-    if (result instanceof Promise) {
-      result.catch(() => {});
     }
   }
 
@@ -438,11 +396,13 @@ return added
         });
       }
 
-      const pubMulti = getRedisPub().multi();
+      const countByProject = new Map<string, number>();
       for (const event of eventsToClickhouse) {
-        await publishEvent('events', 'saved', transformEvent(event), pubMulti);
+        countByProject.set(event.project_id, (countByProject.get(event.project_id) ?? 0) + 1);
+      }
+      for (const [projectId, count] of countByProject) {
+        publishEvent('events', 'batch', { projectId, count });
       }
-      await pubMulti.exec();
 
       await redis
         .multi()
@@ -502,14 +462,34 @@ return added
     });
   }
 
+  private pruneHeartbeatMap() {
+    const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
+    for (const [key, ts] of this.lastHeartbeat) {
+      if (ts < cutoff) {
+        this.lastHeartbeat.delete(key);
+      }
+    }
+  }
+
   private incrementActiveVisitorCount(
     multi: ReturnType<Redis['multi']>,
     projectId: string,
     profileId: string,
   ) {
+    const key = `${projectId}:${profileId}`;
     const now = Date.now();
+    const last = this.lastHeartbeat.get(key) ?? 0;
+
+    if (now - last < this.heartbeatRefreshMs) {
+      return;
+    }
+
+    this.lastHeartbeat.set(key, now);
     const zsetKey = `live:visitors:${projectId}`;
-    return multi.zadd(zsetKey, now, profileId);
+    const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
+    multi
+      .zadd(zsetKey, now, profileId)
+      .set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
   }
 
   public async getActiveVisitorCount(projectId: string): Promise<number> {

packages/redis/publisher.ts

@@ -10,8 +10,7 @@ export type IPublishChannels = {
     };
   };
   events: {
-    received: IServiceEvent;
+    batch: { projectId: string; count: number };
-    saved: IServiceEvent;
   };
   notification: {
     created: Prisma.NotificationUncheckedCreateInput;

packages/trpc/src/routers/realtime.ts

@@ -1,18 +1,15 @@
-import { z } from 'zod';
-
 import {
-  type EventMeta,
-  TABLE_NAMES,
   ch,
   chQuery,
   clix,
-  db,
   formatClickhouseDate,
-  getEventList,
+  type IClickhouseEvent,
+  TABLE_NAMES,
+  transformEvent,
 } from '@openpanel/db';
-
 import { subMinutes } from 'date-fns';
 import sqlstring from 'sqlstring';
+import { z } from 'zod';
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 export const realtimeRouter = createTRPCRouter({
@@ -25,7 +22,7 @@ export const realtimeRouter = createTRPCRouter({
         long: number;
         lat: number;
       }>(
-        `SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`,
+        `SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
       );
 
       return res;
@@ -33,25 +30,18 @@ export const realtimeRouter = createTRPCRouter({
   activeSessions: protectedProcedure
     .input(z.object({ projectId: z.string() }))
     .query(async ({ input }) => {
-      return getEventList({
+      const rows = await chQuery<IClickhouseEvent>(
-        projectId: input.projectId,
+        `SELECT
-        take: 30,
+          name, session_id, created_at, path, origin, referrer, referrer_name,
-        select: {
+          country, city, region, os, os_version, browser, browser_version,
-          name: true,
+          device
-          path: true,
+        FROM ${TABLE_NAMES.events}
-          origin: true,
+        WHERE project_id = ${sqlstring.escape(input.projectId)}
-          referrer: true,
+          AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}'
-          referrerName: true,
+        ORDER BY created_at DESC
-          referrerType: true,
+        LIMIT 50`
-          country: true,
+      );
-          device: true,
+      return rows.map(transformEvent);
-          os: true,
-          browser: true,
-          createdAt: true,
-          profile: true,
-          meta: true,
-        },
-      });
     }),
   paths: protectedProcedure
     .input(z.object({ projectId: z.string() }))
@@ -76,7 +66,7 @@ export const realtimeRouter = createTRPCRouter({
         .where(
           'created_at',
           '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
         )
         .groupBy(['path', 'origin'])
         .orderBy('count', 'DESC')
@@ -106,7 +96,7 @@ export const realtimeRouter = createTRPCRouter({
         .where(
           'created_at',
           '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
         )
         .groupBy(['referrer_name'])
         .orderBy('count', 'DESC')
@@ -137,7 +127,7 @@ export const realtimeRouter = createTRPCRouter({
         .where(
           'created_at',
           '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
         )
         .groupBy(['country', 'city'])
         .orderBy('count', 'DESC')