feature(dashboard): refactor overview

fix(lint)

.cursorrules

@@ -0,0 +1,3 @@
+- When we write clickhouse queries you should always use the custom query builder we have in 
+  - `./packages/db/src/clickhouse/query-builder.ts`
+  - `./packages/db/src/clickhouse/query-functions.ts`

apps/api/scripts/mock.ts

@@ -17,11 +17,7 @@ interface Track {
   type: 'track';
   payload: {
     name: string;
-    properties: {
+    properties: Record<string, string>;
-      __referrer: string;
-      __path: string;
-      __title: string;
-    };
   };
 }
 
@@ -264,25 +260,159 @@ function insertFakeEvents(events: Event[]) {
 }
 
 async function simultaneousRequests() {
-  const events = require('./mock-basic.json');
+  const sessions: {
-  const screenView = events[0]!;
+    ip: string;
-  const event = JSON.parse(JSON.stringify(events[0]));
+    referrer: string;
-  event.track.payload.name = 'click_button';
+    userAgent: string;
-  delete event.track.payload.properties.__referrer;
+    track: Record<string, string>[];
+  }[] = [
+    {
+      ip: '122.168.1.101',
+      referrer: 'https://www.google.com',
+      userAgent:
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+      track: [
+        { name: 'screen_view', path: '/home' },
+        { name: 'button_click', element: 'signup' },
+        { name: 'screen_view', path: '/pricing' },
+      ],
+    },
+    {
+      ip: '192.168.1.101',
+      referrer: 'https://www.bing.com',
+      userAgent:
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+      track: [{ name: 'screen_view', path: '/landing' }],
+    },
+    {
+      ip: '192.168.1.102',
+      referrer: 'https://www.google.com',
+      userAgent:
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
+      track: [{ name: 'screen_view', path: '/about' }],
+    },
+    {
+      ip: '192.168.1.103',
+      referrer: '',
+      userAgent:
+        'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1',
+      track: [
+        { name: 'screen_view', path: '/home' },
+        { name: 'form_submit', form: 'contact' },
+      ],
+    },
+    {
+      ip: '192.168.1.104',
+      referrer: '',
+      userAgent:
+        'Mozilla/5.0 (Linux; Android 11; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36',
+      track: [{ name: 'screen_view', path: '/products' }],
+    },
+    {
+      ip: '203.0.113.101',
+      referrer: 'https://www.facebook.com',
+      userAgent:
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0',
+      track: [
+        { name: 'video_play', videoId: 'abc123' },
+        { name: 'button_click', element: 'subscribe' },
+      ],
+    },
+    {
+      ip: '203.0.113.55',
+      referrer: 'https://www.twitter.com',
+      userAgent:
+        'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1',
+      track: [
+        { name: 'screen_view', path: '/blog' },
+        { name: 'scroll', depth: '50%' },
+      ],
+    },
+    {
+      ip: '198.51.100.20',
+      referrer: 'https://www.linkedin.com',
+      userAgent:
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.902.62 Safari/537.36 Edg/92.0.902.62',
+      track: [{ name: 'button_click', element: 'download' }],
+    },
+    {
+      ip: '198.51.100.21',
+      referrer: 'https://www.google.com',
+      userAgent:
+        'Mozilla/5.0 (Linux; Android 10; SM-A505FN) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36',
+      track: [
+        { name: 'screen_view', path: '/services' },
+        { name: 'button_click', element: 'learn_more' },
+      ],
+    },
+    {
+      ip: '203.0.113.60',
+      referrer: '',
+      userAgent:
+        'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15A5341f Safari/604.1',
+      track: [{ name: 'form_submit', form: 'feedback' }],
+    },
+    {
+      ip: '208.22.132.143',
+      referrer: '',
+      userAgent:
+        'Mozilla/5.0 (Linux; arm_64; Android 10; MAR-LX1H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 YaBrowser/20.4.4.24.00 (alpha) SA/0 Mobile Safari/537.36',
+      track: [
+        { name: 'screen_view', path: '/landing' },
+        { name: 'screen_view', path: '/pricing' },
+        { name: 'screen_view', path: '/blog' },
+        { name: 'screen_view', path: '/blog/post-1' },
+        { name: 'screen_view', path: '/blog/post-2' },
+        { name: 'screen_view', path: '/blog/post-3' },
+        { name: 'screen_view', path: '/blog/post-4' },
+      ],
+    },
+    {
+      ip: '34.187.95.236',
+      referrer: 'https://chatgpt.com',
+      userAgent:
+        'Mozilla/5.0 (Linux; U; Android 9; ar-eg; Redmi 7 Build/PKQ1.181021.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.141 Mobile Safari/537.36 XiaoMi/MiuiBrowser/12.8.3-gn',
+      track: [
+        { name: 'screen_view', path: '/blog' },
+        { name: 'screen_view', path: '/blog/post-1' },
+      ],
+    },
+  ];
 
-  await Promise.all([
+  const screenView: Event = {
-    trackit(event),
+    headers: {
-    trackit({
+      'openpanel-client-id': 'ef38d50e-7d8e-4041-9c62-46d4c3b3bb01',
-      ...event,
+      'x-client-ip': '',
+      'user-agent': '',
+      origin: 'https://openpanel.dev',
+    },
     track: {
-        ...event.track,
+      type: 'track',
       payload: {
-          ...event.track.payload,
+        name: 'screen_view',
-          name: 'text',
+        properties: {},
       },
     },
-    }),
+  };
-  ]);
+
+  for (const session of sessions) {
+    for (const track of session.track) {
+      const { name, ...properties } = track;
+      screenView.track.payload.name = name ?? '';
+      screenView.track.payload.properties.__referrer = session.referrer ?? '';
+      if (name === 'screen_view') {
+        screenView.track.payload.properties.__path =
+          (screenView.headers.origin ?? '') + (properties.path ?? '');
+      } else {
+        screenView.track.payload.name = track.name ?? '';
+        screenView.track.payload.properties = properties;
+      }
+      screenView.headers['x-client-ip'] = session.ip;
+      screenView.headers['user-agent'] = session.userAgent;
+      await trackit(screenView);
+      await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
+    }
+  }
 }
 const exit = async () => {
   await new Promise((resolve) => setTimeout(resolve, 2000));

apps/api/src/index.ts

@@ -41,6 +41,8 @@ import { logger } from './utils/logger';
 
 sourceMapSupport.install();
 
+process.env.TZ = 'UTC';
+
 declare module 'fastify' {
   interface FastifyRequest {
     client: IServiceClientWithProject | null;

apps/dashboard/package.json

@@ -51,6 +51,7 @@
     "@tailwindcss/typography": "^0.5.15",
     "@tanstack/react-query": "^4.36.1",
     "@tanstack/react-table": "^8.11.8",
+    "@tanstack/react-virtual": "^3.13.2",
     "@trpc/client": "^10.45.2",
     "@trpc/next": "^10.45.2",
     "@trpc/react-query": "^10.45.2",
@@ -113,9 +114,9 @@
     "zod": "^3.22.4"
   },
   "devDependencies": {
+    "@openpanel/payments": "workspace:*",
     "@openpanel/trpc": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",
-    "@openpanel/payments": "workspace:*",
     "@types/bcrypt": "^5.0.2",
     "@types/lodash.debounce": "^4.0.9",
     "@types/lodash.isequal": "^4.5.8",

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/conversions.tsx

@@ -9,11 +9,12 @@ type Props = {
 };
 
 const Conversions = ({ projectId }: Props) => {
-  const query = api.event.conversions.useQuery(
+  const query = api.event.conversions.useInfiniteQuery(
     {
       projectId,
     },
     {
+      getNextPageParam: (lastPage) => lastPage.meta.next,
       keepPreviousData: true,
     },
   );

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/events.tsx

@@ -5,13 +5,13 @@ import EventListener from '@/components/events/event-listener';
 import { EventsTable } from '@/components/events/table';
 import { OverviewFiltersButtons } from '@/components/overview/filters/overview-filters-buttons';
 import { OverviewFiltersDrawer } from '@/components/overview/filters/overview-filters-drawer';
-import {
+import { Button } from '@/components/ui/button';
-  useEventQueryFilters,
+import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
-  useEventQueryNamesFilter,
+import { pushModal } from '@/modals';
-} from '@/hooks/useEventQueryFilters';
 import { api } from '@/trpc/client';
-import { Loader2Icon } from 'lucide-react';
+import { format } from 'date-fns';
-import { parseAsInteger, useQueryState } from 'nuqs';
+import { CalendarIcon, Loader2Icon } from 'lucide-react';
+import { parseAsIsoDateTime, useQueryState } from 'nuqs';
 
 type Props = {
   projectId: string;
@@ -20,21 +20,22 @@ type Props = {
 
 const Events = ({ projectId, profileId }: Props) => {
   const [filters] = useEventQueryFilters();
-  const [eventNames] = useEventQueryNamesFilter();
+  const [startDate, setStartDate] = useQueryState(
-  const [cursor, setCursor] = useQueryState(
+    'startDate',
-    'cursor',
+    parseAsIsoDateTime,
-    parseAsInteger.withDefault(0),
   );
-  const query = api.event.events.useQuery(
+
+  const [endDate, setEndDate] = useQueryState('endDate', parseAsIsoDateTime);
+  const query = api.event.events.useInfiniteQuery(
     {
-      cursor,
       projectId,
-      take: 50,
-      events: eventNames,
       filters,
       profileId,
+      startDate: startDate || undefined,
+      endDate: endDate || undefined,
     },
     {
+      getNextPageParam: (lastPage) => lastPage.meta.next,
       keepPreviousData: true,
     },
   );
@@ -43,6 +44,25 @@ const Events = ({ projectId, profileId }: Props) => {
     <div>
       <TableButtons>
         <EventListener onRefresh={() => query.refetch()} />
+        <Button
+          variant="outline"
+          size="sm"
+          icon={CalendarIcon}
+          onClick={() => {
+            pushModal('DateRangerPicker', {
+              onChange: ({ startDate, endDate }) => {
+                setStartDate(startDate);
+                setEndDate(endDate);
+              },
+              startDate: startDate || undefined,
+              endDate: endDate || undefined,
+            });
+          }}
+        >
+          {startDate && endDate
+            ? `${format(startDate, 'MMM d')} - ${format(endDate, 'MMM d')}`
+            : 'Date range'}
+        </Button>
         <OverviewFiltersDrawer
           mode="events"
           projectId={projectId}
@@ -58,7 +78,7 @@ const Events = ({ projectId, profileId }: Props) => {
           </div>
         )}
       </TableButtons>
-      <EventsTable query={query} cursor={cursor} setCursor={setCursor} />
+      <EventsTable query={query} />
     </div>
   );
 };

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/page.tsx

@@ -1,7 +1,6 @@
 import { OverviewFiltersButtons } from '@/components/overview/filters/overview-filters-buttons';
 import { OverviewFiltersDrawer } from '@/components/overview/filters/overview-filters-drawer';
 import ServerLiveCounter from '@/components/overview/live-counter';
-import OverviewMetrics from '@/components/overview/overview-metrics';
 import OverviewShareServer from '@/components/overview/overview-share';
 import OverviewTopDevices from '@/components/overview/overview-top-devices';
 import OverviewTopEvents from '@/components/overview/overview-top-events';
@@ -10,6 +9,7 @@ import OverviewTopPages from '@/components/overview/overview-top-pages';
 import OverviewTopSources from '@/components/overview/overview-top-sources';
 
 import { OverviewInterval } from '@/components/overview/overview-interval';
+import OverviewMetricsV2 from '@/components/overview/overview-metrics-v2';
 import { OverviewRange } from '@/components/overview/overview-range';
 
 interface PageProps {
@@ -36,7 +36,8 @@ export default function Page({ params: { projectId } }: PageProps) {
         <OverviewFiltersButtons />
       </div>
       <div className="grid grid-cols-6 gap-4 p-4 pt-0">
-        <OverviewMetrics projectId={projectId} />
+        {/* <OverviewMetrics projectId={projectId} /> */}
+        <OverviewMetricsV2 projectId={projectId} />
         <OverviewTopSources projectId={projectId} />
         <OverviewTopPages projectId={projectId} />
         <OverviewTopDevices projectId={projectId} />

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-events.tsx

@@ -4,15 +4,9 @@ import { TableButtons } from '@/components/data-table';
 import { EventsTable } from '@/components/events/table';
 import { OverviewFiltersButtons } from '@/components/overview/filters/overview-filters-buttons';
 import { OverviewFiltersDrawer } from '@/components/overview/filters/overview-filters-drawer';
-import {
+import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
-  useEventQueryFilters,
-  useEventQueryNamesFilter,
-} from '@/hooks/useEventQueryFilters';
 import { api } from '@/trpc/client';
 import { Loader2Icon } from 'lucide-react';
-import { parseAsInteger, useQueryState } from 'nuqs';
-
-import { GetEventListOptions } from '@openpanel/db';
 
 type Props = {
   projectId: string;
@@ -21,21 +15,14 @@ type Props = {
 
 const Events = ({ projectId, profileId }: Props) => {
   const [filters] = useEventQueryFilters();
-  const [eventNames] = useEventQueryNamesFilter();
+  const query = api.event.events.useInfiniteQuery(
-  const [cursor, setCursor] = useQueryState(
-    'cursor',
-    parseAsInteger.withDefault(0),
-  );
-  const query = api.event.events.useQuery(
     {
-      cursor,
       projectId,
-      take: 50,
-      events: eventNames,
       filters,
       profileId,
     },
     {
+      getNextPageParam: (lastPage) => lastPage.meta.next,
       keepPreviousData: true,
     },
   );
@@ -58,7 +45,7 @@ const Events = ({ projectId, profileId }: Props) => {
           </div>
         )}
       </TableButtons>
-      <EventsTable query={query} cursor={cursor} setCursor={setCursor} />
+      <EventsTable query={query} />
     </div>
   );
 };

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/map.tsx

@@ -49,24 +49,13 @@ const Map = ({ markers }: Props) => {
   const boundingBox = getBoundingBox(hull);
   const [zoom] = useAnimatedState(
     markers.length === 1
-      ? 20
+      ? 1
       : determineZoom(boundingBox, size ? size?.height / size?.width : 1),
   );
 
   const [long] = useAnimatedState(center.long);
   const [lat] = useAnimatedState(center.lat);
 
-  useEffect(() => {
-    requestAnimationFrame(() => {
-      if (ref.current) {
-        setSize({
-          width: ref.current.clientWidth,
-          height: ref.current.clientHeight,
-        });
-      }
-    });
-  }, [isFullscreen]);
-
   useEffect(() => {
     return bind(window, {
       type: 'resize',
@@ -95,20 +84,12 @@ const Map = ({ markers }: Props) => {
   const theme = useTheme();
 
   return (
-    <div
+    <div className={cn('absolute bottom-0 left-0 right-0 top-0')} ref={ref}>
-      className={cn(
-        'fixed bottom-0 left-0 right-0 top-0',
-        !isFullscreen && 'lg:left-72',
-      )}
-      ref={ref}
-    >
       {size === null ? (
         <></>
       ) : (
         <>
           <ComposableMap
-            width={size?.width}
-            height={size?.height}
             projection="geoMercator"
             projectionConfig={{
               rotate: [0, 0, 0],

apps/dashboard/src/components/charts/chart-tooltip.tsx

@@ -0,0 +1,59 @@
+import { createContext, useContext as useBaseContext } from 'react';
+
+import { Tooltip as RechartsTooltip, type TooltipProps } from 'recharts';
+
+export function createChartTooltip<
+  PropsFromTooltip extends Record<string, unknown>,
+  PropsFromContext extends Record<string, unknown>,
+>(
+  Tooltip: React.ComponentType<
+    {
+      context: PropsFromContext;
+      data: PropsFromTooltip;
+    } & TooltipProps<number, string>
+  >,
+) {
+  const context = createContext<PropsFromContext | null>(null);
+  const useContext = () => {
+    const value = useBaseContext(context);
+    if (!value) {
+      throw new Error('ChartTooltip context not found');
+    }
+    return value;
+  };
+
+  const InnerTooltip = (tooltip: TooltipProps<number, string>) => {
+    const context = useContext();
+    const data = tooltip.payload?.[0]?.payload;
+
+    if (!data || !tooltip.active) {
+      return null;
+    }
+
+    return (
+      <div className="flex min-w-[180px] flex-col gap-2 rounded-xl border bg-background/80 p-3  shadow-xl backdrop-blur-sm">
+        <Tooltip data={data} context={context} {...tooltip} />
+      </div>
+    );
+  };
+
+  return {
+    TooltipProvider: ({
+      children,
+      ...value
+    }: {
+      children: React.ReactNode;
+    } & PropsFromContext) => {
+      return (
+        <context.Provider value={value as unknown as PropsFromContext}>
+          {children}
+        </context.Provider>
+      );
+    },
+    Tooltip: (props: TooltipProps<number, string>) => {
+      return (
+        <RechartsTooltip {...props} content={<InnerTooltip {...props} />} />
+      );
+    },
+  };
+}

apps/dashboard/src/components/events/event-list-item.tsx

@@ -56,6 +56,8 @@ export function EventListItem(props: EventListItemProps) {
           if (!isMinimal) {
             pushModal('EventDetails', {
               id: props.id,
+              projectId,
+              createdAt,
             });
           }
         }}

apps/dashboard/src/components/events/table/columns.tsx

@@ -15,6 +15,7 @@ export function useColumns() {
   const number = useNumber();
   const columns: ColumnDef<IServiceEvent>[] = [
     {
+      size: 300,
       accessorKey: 'name',
       header: 'Name',
       cell({ row }) {
@@ -50,7 +51,6 @@ export function useColumns() {
 
         return (
           <div className="flex items-center gap-2">
-            <TooltipComplete content="Click to edit" side="left">
             <button
               type="button"
               className="transition-transform hover:scale-105"
@@ -66,13 +66,14 @@ export function useColumns() {
                 meta={row.original.meta}
               />
             </button>
-            </TooltipComplete>
             <span className="flex gap-2">
               <button
                 type="button"
                 onClick={() => {
                   pushModal('EventDetails', {
                     id: row.original.id,
+                    createdAt: row.original.createdAt,
+                    projectId: row.original.projectId,
                   });
                 }}
                 className="font-medium"
@@ -86,41 +87,13 @@ export function useColumns() {
       },
     },
     {
-      accessorKey: 'country',
+      accessorKey: 'createdAt',
-      header: 'Country',
+      header: 'Created at',
+      size: 170,
       cell({ row }) {
-        const { country, city } = row.original;
+        const date = row.original.createdAt;
         return (
-          <div className="inline-flex min-w-full flex-none items-center gap-2">
+          <div>{isToday(date) ? formatTime(date) : formatDateTime(date)}</div>
-            <SerieIcon name={country} />
-            <span>{city}</span>
-          </div>
-        );
-      },
-    },
-    {
-      accessorKey: 'os',
-      header: 'OS',
-      cell({ row }) {
-        const { os } = row.original;
-        return (
-          <div className="flex min-w-full items-center gap-2">
-            <SerieIcon name={os} />
-            <span>{os}</span>
-          </div>
-        );
-      },
-    },
-    {
-      accessorKey: 'browser',
-      header: 'Browser',
-      cell({ row }) {
-        const { browser } = row.original;
-        return (
-          <div className="inline-flex min-w-full flex-none items-center gap-2">
-            <SerieIcon name={browser} />
-            <span>{browser}</span>
-          </div>
         );
       },
     },
@@ -134,8 +107,8 @@ export function useColumns() {
         }
         return (
           <ProjectLink
-            href={`/profiles/${profile?.id}`}
+            href={`/profiles/${profile.id}`}
-            className="max-w-[80px] overflow-hidden text-ellipsis whitespace-nowrap font-medium hover:underline"
+            className="whitespace-nowrap font-medium hover:underline"
           >
             {getProfileName(profile)}
           </ProjectLink>
@@ -143,12 +116,44 @@ export function useColumns() {
       },
     },
     {
-      accessorKey: 'createdAt',
+      accessorKey: 'country',
-      header: 'Created at',
+      header: 'Country',
+      size: 150,
       cell({ row }) {
-        const date = row.original.createdAt;
+        const { country, city } = row.original;
         return (
-          <div>{isToday(date) ? formatTime(date) : formatDateTime(date)}</div>
+          <div className="row items-center gap-2 min-w-0">
+            <SerieIcon name={country} />
+            <span className="truncate">{city}</span>
+          </div>
+        );
+      },
+    },
+    {
+      accessorKey: 'os',
+      header: 'OS',
+      size: 130,
+      cell({ row }) {
+        const { os } = row.original;
+        return (
+          <div className="row items-center gap-2 min-w-0">
+            <SerieIcon name={os} />
+            <span className="truncate">{os}</span>
+          </div>
+        );
+      },
+    },
+    {
+      accessorKey: 'browser',
+      header: 'Browser',
+      size: 110,
+      cell({ row }) {
+        const { browser } = row.original;
+        return (
+          <div className="row items-center gap-2 min-w-0">
+            <SerieIcon name={browser} />
+            <span className="truncate">{browser}</span>
+          </div>
         );
       },
     },

apps/dashboard/src/components/events/table/events-data-table.tsx

@@ -0,0 +1,152 @@
+'use client';
+
+import { GridCell } from '@/components/grid-table';
+import { cn } from '@/utils/cn';
+import {
+  flexRender,
+  getCoreRowModel,
+  useReactTable,
+} from '@tanstack/react-table';
+import type { ColumnDef } from '@tanstack/react-table';
+import { useVirtualizer, useWindowVirtualizer } from '@tanstack/react-virtual';
+import throttle from 'lodash.throttle';
+import { useEffect, useRef, useState } from 'react';
+
+interface DataTableProps<TData> {
+  columns: ColumnDef<TData, any>[];
+  data: TData[];
+}
+
+export function EventsDataTable<TData>({
+  columns,
+  data,
+}: DataTableProps<TData>) {
+  const table = useReactTable({
+    data,
+    columns,
+    getCoreRowModel: getCoreRowModel(),
+  });
+
+  const parentRef = useRef<HTMLDivElement>(null);
+  const [scrollMargin, setScrollMargin] = useState(0);
+  const { rows } = table.getRowModel();
+
+  const virtualizer = useWindowVirtualizer({
+    count: rows.length,
+    estimateSize: () => 48,
+    scrollMargin,
+    overscan: 10,
+  });
+
+  useEffect(() => {
+    const updateScrollMargin = throttle(() => {
+      if (parentRef.current) {
+        setScrollMargin(
+          parentRef.current.getBoundingClientRect().top + window.scrollY,
+        );
+      }
+    }, 500);
+
+    // Initial calculation
+    updateScrollMargin();
+
+    // Listen for scroll and resize events
+    window.addEventListener('scroll', updateScrollMargin);
+    window.addEventListener('resize', updateScrollMargin);
+
+    return () => {
+      window.removeEventListener('scroll', updateScrollMargin);
+      window.removeEventListener('resize', updateScrollMargin);
+    };
+  }, []); // Empty dependency array since we're setting up listeners
+
+  const visibleRows = virtualizer.getVirtualItems();
+
+  return (
+    <div className="card">
+      <div className="relative w-full overflow-auto rounded-md">
+        <div
+          className="w-full"
+          style={{
+            width: 'max-content',
+            minWidth: '100%',
+          }}
+        >
+          {table.getHeaderGroups().map((headerGroup) => (
+            <div className="thead row h-12 sticky top-0" key={headerGroup.id}>
+              {headerGroup.headers.map((header) => {
+                return (
+                  <GridCell
+                    key={header.id}
+                    isHeader
+                    style={{
+                      minWidth: header.column.getSize(),
+                      flexShrink: 1,
+                      overflow: 'hidden',
+                      flex: 1,
+                    }}
+                  >
+                    {header.isPlaceholder
+                      ? null
+                      : flexRender(
+                          header.column.columnDef.header,
+                          header.getContext(),
+                        )}
+                  </GridCell>
+                );
+              })}
+            </div>
+          ))}
+          <div ref={parentRef} className="w-full">
+            <div
+              className="tbody [&>*:last-child]:border-0"
+              style={{
+                height: `${virtualizer.getTotalSize()}px`,
+                width: '100%',
+                position: 'relative',
+              }}
+            >
+              {visibleRows.map((virtualRow, index) => {
+                const row = rows[virtualRow.index]!;
+                if (!row) {
+                  return null;
+                }
+
+                return (
+                  <div
+                    key={row.id}
+                    className={cn('absolute top-0 left-0 w-full h-12 row')}
+                    style={{
+                      transform: `translateY(${
+                        virtualRow.start - virtualizer.options.scrollMargin
+                      }px)`,
+                    }}
+                  >
+                    {row.getVisibleCells().map((cell) => {
+                      return (
+                        <GridCell
+                          key={cell.id}
+                          style={{
+                            minWidth: cell.column.getSize(),
+                            flexShrink: 1,
+                            overflow: 'hidden',
+                            flex: 1,
+                          }}
+                        >
+                          {flexRender(
+                            cell.column.columnDef.cell,
+                            cell.getContext(),
+                          )}
+                        </GridCell>
+                      );
+                    })}
+                  </div>
+                );
+              })}
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}

apps/dashboard/src/components/events/table/index.tsx

@@ -1,64 +1,82 @@
-import { DataTable } from '@/components/data-table';
 import { FullPageEmptyState } from '@/components/full-page-empty-state';
-import { Pagination } from '@/components/pagination';
-import { Button } from '@/components/ui/button';
 import { TableSkeleton } from '@/components/ui/table';
-import type { UseQueryResult } from '@tanstack/react-query';
+import type {
-import { GanttChartIcon } from 'lucide-react';
+  UseInfiniteQueryResult,
-import { column } from 'mathjs';
+  UseQueryResult,
-import type { Dispatch, SetStateAction } from 'react';
+} from '@tanstack/react-query';
-
+import { GanttChartIcon, Loader2Icon } from 'lucide-react';
-import type { IServiceEvent } from '@openpanel/db';
+import { useEffect, useRef } from 'react';
 
+import type { RouterOutputs } from '@/trpc/client';
+import { cn } from '@/utils/cn';
+import { useInViewport } from 'react-in-viewport';
 import { useColumns } from './columns';
+import { EventsDataTable } from './events-data-table';
 
 type Props =
   | {
-      query: UseQueryResult<IServiceEvent[]>;
+      query: UseInfiniteQueryResult<RouterOutputs['event']['events']>;
     }
   | {
-      query: UseQueryResult<IServiceEvent[]>;
+      query: UseQueryResult<RouterOutputs['event']['events']>;
-      cursor: number;
-      setCursor: Dispatch<SetStateAction<number>>;
     };
 
 export const EventsTable = ({ query, ...props }: Props) => {
   const columns = useColumns();
-  const { data, isFetching, isLoading } = query;
+  const { isLoading } = query;
+  const ref = useRef<HTMLDivElement>(null);
+  const { inViewport, enterCount } = useInViewport(ref, undefined, {
+    disconnectOnLeave: true,
+  });
+  const isInfiniteQuery = 'fetchNextPage' in query;
+  const data =
+    (isInfiniteQuery
+      ? query.data?.pages?.flatMap((p) => p.items)
+      : query.data?.items) ?? [];
+
+  const hasNextPage = isInfiniteQuery
+    ? query.data?.pages[query.data.pages.length - 1]?.meta.next
+    : query.data?.meta.next;
+
+  useEffect(() => {
+    if (
+      hasNextPage &&
+      isInfiniteQuery &&
+      data.length > 0 &&
+      inViewport &&
+      enterCount > 0 &&
+      query.isFetchingNextPage === false
+    ) {
+      query.fetchNextPage();
+    }
+  }, [inViewport, enterCount, hasNextPage]);
 
   if (isLoading) {
     return <TableSkeleton cols={columns.length} />;
   }
 
-  if (data?.length === 0) {
+  if (data.length === 0) {
     return (
       <FullPageEmptyState title="No events here" icon={GanttChartIcon}>
         <p>Could not find any events</p>
-        {'cursor' in props && props.cursor !== 0 && (
-          <Button
-            className="mt-8"
-            variant="outline"
-            onClick={() => props.setCursor((p) => p - 1)}
-          >
-            Go to previous page
-          </Button>
-        )}
       </FullPageEmptyState>
     );
   }
 
   return (
     <>
-      <DataTable data={data ?? []} columns={columns} />
+      <EventsDataTable data={data} columns={columns} />
-      {'cursor' in props && (
+      {isInfiniteQuery && (
-        <Pagination
+        <div className="w-full h-10 center-center pt-10" ref={ref}>
-          className="mt-2"
+          <div
-          setCursor={props.setCursor}
+            className={cn(
-          cursor={props.cursor}
+              'size-8 bg-background rounded-full center-center border opacity-0 transition-opacity',
-          count={Number.POSITIVE_INFINITY}
+              isInfiniteQuery && query.isFetchingNextPage && 'opacity-100',
-          take={50}
+            )}
-          loading={isFetching}
+          >
-        />
+            <Loader2Icon className="size-4 animate-spin" />
+          </div>
+        </div>
       )}
     </>
   );

apps/dashboard/src/components/forms/copy-input.tsx

@@ -18,7 +18,7 @@ const CopyInput = ({ label, value, className }: Props) => {
       onClick={() => clipboard(value)}
     >
       {!!label && <Label>{label}</Label>}
-      <div className="font-mono flex items-center justify-between rounded border-input bg-def-200 p-2 px-3 ">
+      <div className="font-mono flex items-center justify-between rounded border-input bg-card p-2 px-3 ">
         {value}
         <CopyIcon size={16} />
       </div>

apps/dashboard/src/components/forms/tag-input.tsx

@@ -138,7 +138,7 @@ const TagInput = ({
       <input
         ref={inputRef}
         placeholder={`${placeholder} ↵`}
-        className="min-w-20 flex-1 py-1  focus-visible:outline-none"
+        className="min-w-20 flex-1 py-1  focus-visible:outline-none bg-card"
         value={inputValue}
         onChange={(e) => setInputValue(e.target.value)}
         onKeyDown={handleKeyDown}

apps/dashboard/src/components/grid-table.tsx

@@ -66,7 +66,7 @@ export const GridCell: React.FC<
     )}
     {...props}
   >
-    {children}
+    <div className="truncate w-full">{children}</div>
   </Component>
 );
 

apps/dashboard/src/components/overview/filters/overview-filters-buttons.tsx

@@ -7,6 +7,7 @@ import {
 } from '@/hooks/useEventQueryFilters';
 import { getPropertyLabel } from '@/translations/properties';
 import { cn } from '@/utils/cn';
+import { operators } from '@openpanel/constants';
 import { X } from 'lucide-react';
 import type { Options as NuqsOptions } from 'nuqs';
 
@@ -20,7 +21,8 @@ export function OverviewFiltersButtons({
   nuqsOptions,
 }: OverviewFiltersButtonsProps) {
   const [events, setEvents] = useEventQueryNamesFilter(nuqsOptions);
-  const [filters, setFilter] = useEventQueryFilters(nuqsOptions);
+  const [filters, setFilter, setFilters, removeFilter] =
+    useEventQueryFilters(nuqsOptions);
   if (filters.length === 0 && events.length === 0) return null;
   return (
     <div className={cn('flex flex-wrap gap-2', className)}>
@@ -36,20 +38,23 @@ export function OverviewFiltersButtons({
         </Button>
       ))}
       {filters.map((filter) => {
-        if (!filter.value[0]) {
-          return null;
-        }
-
         return (
           <Button
             key={filter.name}
             size="sm"
             variant="outline"
             icon={X}
-            onClick={() => setFilter(filter.name, [], 'is')}
+            onClick={() => removeFilter(filter.name)}
           >
-            <span className="mr-1">{getPropertyLabel(filter.name)} is</span>
+            <span>{getPropertyLabel(filter.name)}</span>
-            <strong className="font-semibold">{filter.value.join(', ')}</strong>
+            <span className="opacity-40 ml-2 lowercase">
+              {operators[filter.operator]}
+            </span>
+            {filter.value.length > 0 && (
+              <strong className="font-semibold ml-2">
+                {filter.value.join(', ')}
+              </strong>
+            )}
           </Button>
         );
       })}

apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx

@@ -31,6 +31,10 @@ export interface OverviewFiltersDrawerContentProps {
   mode: 'profiles' | 'events';
 }
 
+const excludePropertyFilter = (name: string) => {
+  return ['*', 'duration', 'created_at', 'has_profile'].includes(name);
+};
+
 export function OverviewFiltersDrawerContent({
   projectId,
   nuqsOptions,
@@ -60,7 +64,9 @@ export function OverviewFiltersDrawerContent({
               value={event}
               onChange={setEvent}
               // First items is * which is only used for report editing
-              items={eventNames.slice(1).map((item) => ({
+              items={eventNames
+                .filter((item) => !excludePropertyFilter(item.name))
+                .map((item) => ({
                   label: item.name,
                   value: item.name,
                 }))}

apps/dashboard/src/components/overview/overview-constants.tsx

@@ -0,0 +1,49 @@
+import type { IGetTopGenericInput } from '@openpanel/db';
+
+export const OVERVIEW_COLUMNS_NAME: Record<
+  IGetTopGenericInput['column'],
+  string
+> = {
+  country: 'Country',
+  region: 'Region',
+  city: 'City',
+  browser: 'Browser',
+  brand: 'Brand',
+  os: 'OS',
+  device: 'Device',
+  browser_version: 'Browser version',
+  os_version: 'OS version',
+  model: 'Model',
+  referrer: 'Referrer',
+  referrer_name: 'Referrer name',
+  referrer_type: 'Referrer type',
+  utm_source: 'UTM source',
+  utm_medium: 'UTM medium',
+  utm_campaign: 'UTM campaign',
+  utm_term: 'UTM term',
+  utm_content: 'UTM content',
+};
+
+export const OVERVIEW_COLUMNS_NAME_PLURAL: Record<
+  IGetTopGenericInput['column'],
+  string
+> = {
+  country: 'Countries',
+  region: 'Regions',
+  city: 'Cities',
+  browser: 'Browsers',
+  brand: 'Brands',
+  os: 'OSs',
+  device: 'Devices',
+  browser_version: 'Browser versions',
+  os_version: 'OS versions',
+  model: 'Models',
+  referrer: 'Referrers',
+  referrer_name: 'Referrer names',
+  referrer_type: 'Referrer types',
+  utm_source: 'UTM sources',
+  utm_medium: 'UTM mediums',
+  utm_campaign: 'UTM campaigns',
+  utm_term: 'UTM terms',
+  utm_content: 'UTM contents',
+};

apps/dashboard/src/components/overview/overview-details-button.tsx

@@ -1,25 +1,12 @@
-import { pushModal } from '@/modals';
 import { ScanEyeIcon } from 'lucide-react';
 
-import type { IChartProps } from '@openpanel/validation';
+import { Button, type ButtonProps } from '../ui/button';
 
-import { Button } from '../ui/button';
+type Props = Omit<ButtonProps, 'children'>;
 
-type Props = {
+const OverviewDetailsButton = (props: Props) => {
-  chart: IChartProps;
-};
-
-const OverviewDetailsButton = ({ chart }: Props) => {
   return (
-    <Button
+    <Button size="icon" variant="ghost" {...props}>
-      size="icon"
-      variant="ghost"
-      onClick={() => {
-        pushModal('OverviewChartDetails', {
-          chart: chart,
-        });
-      }}
-    >
       <ScanEyeIcon size={18} />
     </Button>
   );

apps/dashboard/src/components/overview/overview-metric-card.tsx

@@ -0,0 +1,192 @@
+'use client';
+
+import { fancyMinutes, useNumber } from '@/hooks/useNumerFormatter';
+import type { IChartData, RouterOutputs } from '@/trpc/client';
+import { cn } from '@/utils/cn';
+import AutoSizer from 'react-virtualized-auto-sizer';
+import { Area, AreaChart } from 'recharts';
+
+import { average, getPreviousMetric, sum } from '@openpanel/common';
+import type { IChartMetric, Metrics } from '@openpanel/validation';
+import {
+  PreviousDiffIndicator,
+  PreviousDiffIndicatorPure,
+  getDiffIndicator,
+} from '../report-chart/common/previous-diff-indicator';
+import { Skeleton } from '../skeleton';
+import { Tooltiper } from '../ui/tooltip';
+
+interface MetricCardProps {
+  id: string;
+  data: {
+    current: number;
+    previous?: number;
+  }[];
+  metric: {
+    current: number;
+    previous?: number | null;
+  };
+  unit?: string;
+  label: string;
+  onClick?: () => void;
+  active?: boolean;
+  inverted?: boolean;
+  isLoading?: boolean;
+}
+
+export function OverviewMetricCard({
+  id,
+  data,
+  metric,
+  unit,
+  label,
+  onClick,
+  active,
+  inverted = false,
+  isLoading = false,
+}: MetricCardProps) {
+  const number = useNumber();
+  const { current, previous } = metric;
+
+  const renderValue = (value: number, unitClassName?: string, short = true) => {
+    if (unit === 'min') {
+      return <>{fancyMinutes(value)}</>;
+    }
+
+    return (
+      <>
+        {short ? number.short(value) : number.format(value)}
+        {unit && <span className={unitClassName}>{unit}</span>}
+      </>
+    );
+  };
+
+  const graphColors = getDiffIndicator(
+    inverted,
+    getPreviousMetric(current, previous)?.state,
+    '#6ee7b7', // green
+    '#fda4af', // red
+    '#93c5fd', // blue
+  );
+
+  return (
+    <Tooltiper
+      content={
+        <span>
+          {label}:{' '}
+          <span className="font-semibold">
+            {renderValue(current, 'ml-1 font-light text-xl', false)}
+          </span>
+        </span>
+      }
+      asChild
+      sideOffset={-20}
+    >
+      <button
+        type="button"
+        className={cn(
+          'col-span-2 flex-1 shadow-[0_0_0_0.5px] shadow-border md:col-span-1',
+          active && 'bg-def-100',
+        )}
+        onClick={onClick}
+      >
+        <div className={cn('group relative p-4')}>
+          <div
+            className={cn(
+              'pointer-events-none absolute -left-1 -right-1 bottom-0 top-0 z-0 opacity-50 transition-opacity duration-300 group-hover:opacity-100',
+            )}
+          >
+            <AutoSizer>
+              {({ width, height }) => (
+                <AreaChart
+                  width={width}
+                  height={height / 4}
+                  data={data}
+                  style={{ marginTop: (height / 4) * 3 }}
+                >
+                  <defs>
+                    <linearGradient
+                      id={`colorUv${id}`}
+                      x1="0"
+                      y1="0"
+                      x2="0"
+                      y2="1"
+                    >
+                      <stop
+                        offset="0%"
+                        stopColor={graphColors}
+                        stopOpacity={0.2}
+                      />
+                      <stop
+                        offset="100%"
+                        stopColor={graphColors}
+                        stopOpacity={0.05}
+                      />
+                    </linearGradient>
+                  </defs>
+                  <Area
+                    dataKey={'current'}
+                    type="step"
+                    fill={`url(#colorUv${id})`}
+                    fillOpacity={1}
+                    stroke={graphColors}
+                    strokeWidth={1}
+                    isAnimationActive={false}
+                  />
+                </AreaChart>
+              )}
+            </AutoSizer>
+          </div>
+          <OverviewMetricCardNumber
+            label={label}
+            value={renderValue(current, 'ml-1 font-light text-xl')}
+            enhancer={
+              <PreviousDiffIndicatorPure
+                className="text-sm"
+                size="sm"
+                inverted={inverted}
+                {...getPreviousMetric(current, previous)}
+              />
+            }
+            isLoading={isLoading}
+          />
+        </div>
+      </button>
+    </Tooltiper>
+  );
+}
+
+export function OverviewMetricCardNumber({
+  label,
+  value,
+  enhancer,
+  className,
+  isLoading,
+}: {
+  label: React.ReactNode;
+  value: React.ReactNode;
+  enhancer?: React.ReactNode;
+  className?: string;
+  isLoading?: boolean;
+}) {
+  return (
+    <div className={cn('flex min-w-0 flex-col gap-2', className)}>
+      <div className="flex items-center justify-between gap-2">
+        <div className="flex min-w-0 items-center gap-2 text-left">
+          <span className="truncate text-muted-foreground">{label}</span>
+        </div>
+      </div>
+      {isLoading ? (
+        <div className="flex items-end justify-between gap-4">
+          <Skeleton className="h-6 w-16" />
+          <Skeleton className="h-6 w-12" />
+        </div>
+      ) : (
+        <div className="flex items-end justify-between gap-4">
+          <div className="truncate font-mono text-3xl font-bold">{value}</div>
+          {enhancer}
+        </div>
+      )}
+    </div>
+  );
+}

apps/dashboard/src/components/overview/overview-metrics-v2.tsx

@@ -0,0 +1,258 @@
+'use client';
+
+import { useOverviewOptions } from '@/components/overview/useOverviewOptions';
+import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
+import { cn } from '@/utils/cn';
+
+import { useFormatDateInterval } from '@/hooks/useFormatDateInterval';
+import { useNumber } from '@/hooks/useNumerFormatter';
+import { type RouterOutputs, api } from '@/trpc/client';
+import { getChartColor } from '@/utils/theme';
+import { getPreviousMetric } from '@openpanel/common';
+import React from 'react';
+import {
+  CartesianGrid,
+  Line,
+  LineChart,
+  ResponsiveContainer,
+  XAxis,
+  YAxis,
+} from 'recharts';
+import { createChartTooltip } from '../charts/chart-tooltip';
+import { useXAxisProps, useYAxisProps } from '../report-chart/common/axis';
+import { PreviousDiffIndicatorPure } from '../report-chart/common/previous-diff-indicator';
+import { Skeleton } from '../skeleton';
+import { OverviewLiveHistogram } from './overview-live-histogram';
+import { OverviewMetricCard } from './overview-metric-card';
+
+interface OverviewMetricsProps {
+  projectId: string;
+}
+
+const TITLES = [
+  {
+    title: 'Unique Visitors',
+    key: 'unique_visitors',
+    unit: '',
+    inverted: false,
+  },
+  {
+    title: 'Sessions',
+    key: 'total_sessions',
+    unit: '',
+    inverted: false,
+  },
+  {
+    title: 'Pageviews',
+    key: 'total_screen_views',
+    unit: '',
+    inverted: false,
+  },
+  {
+    title: 'Pages per session',
+    key: 'views_per_session',
+    unit: '',
+    inverted: false,
+  },
+  {
+    title: 'Bounce Rate',
+    key: 'bounce_rate',
+    unit: '%',
+    inverted: true,
+  },
+  {
+    title: 'Session Duration',
+    key: 'avg_session_duration',
+    unit: 'min',
+    inverted: false,
+  },
+] as const;
+
+export default function OverviewMetricsV2({ projectId }: OverviewMetricsProps) {
+  const { range, interval, metric, setMetric, startDate, endDate } =
+    useOverviewOptions();
+  const [filters] = useEventQueryFilters();
+
+  const activeMetric = TITLES[metric]!;
+  const overviewQuery = api.overview.stats.useQuery({
+    projectId,
+    range,
+    interval,
+    filters,
+    startDate,
+    endDate,
+  });
+
+  const data =
+    overviewQuery.data?.series?.map((item) => ({
+      ...item,
+      timestamp: new Date(item.date).getTime(),
+    })) || [];
+
+  const xAxisProps = useXAxisProps({ interval: 'day' });
+  const yAxisProps = useYAxisProps();
+
+  return (
+    <>
+      <div className="relative -top-0.5 col-span-6 -m-4 mb-0 mt-0 md:m-0">
+        <div className="card mb-2 grid grid-cols-4 overflow-hidden rounded-md">
+          {TITLES.map((title, index) => (
+            <OverviewMetricCard
+              key={title.key}
+              id={title.key}
+              onClick={() => setMetric(index)}
+              label={title.title}
+              metric={{
+                current: overviewQuery.data?.metrics[title.key] ?? 0,
+                previous: overviewQuery.data?.metrics[`prev_${title.key}`],
+              }}
+              unit={title.unit}
+              data={data.map((item) => ({
+                current: item[title.key],
+                previous: item[`prev_${title.key}`],
+              }))}
+              active={metric === index}
+              isLoading={overviewQuery.isLoading}
+            />
+          ))}
+
+          <div
+            className={cn(
+              'col-span-4 min-h-16 flex-1 p-4 pb-0 shadow-[0_0_0_0.5px] shadow-border max-md:row-start-1 md:col-span-2',
+            )}
+          >
+            <OverviewLiveHistogram projectId={projectId} />
+          </div>
+        </div>
+
+        <div className="card p-4">
+          <div className="text-center text-muted-foreground mb-2">
+            {activeMetric.title}
+          </div>
+          <div className="w-full h-[150px]">
+            {overviewQuery.isLoading && <Skeleton className="h-full w-full" />}
+            <TooltipProvider metric={activeMetric}>
+              <ResponsiveContainer width="100%" height="100%">
+                <LineChart data={data}>
+                  <Tooltip />
+                  <YAxis
+                    {...yAxisProps}
+                    domain={[
+                      0,
+                      activeMetric.key === 'bounce_rate' ? 100 : 'dataMax',
+                    ]}
+                    width={25}
+                  />
+                  <XAxis {...xAxisProps} />
+
+                  <CartesianGrid
+                    strokeDasharray="3 3"
+                    horizontal={true}
+                    vertical={false}
+                    className="stroke-border"
+                  />
+
+                  <Line
+                    key={`prev_${activeMetric.key}`}
+                    type="linear"
+                    dataKey={`prev_${activeMetric.key}`}
+                    stroke={'hsl(var(--foreground) / 0.2)'}
+                    strokeWidth={2}
+                    isAnimationActive={false}
+                    dot={
+                      data.length > 90
+                        ? false
+                        : {
+                            stroke: 'hsl(var(--foreground) / 0.2)',
+                            fill: 'hsl(var(--def-100))',
+                            strokeWidth: 1.5,
+                            r: 3,
+                          }
+                    }
+                    activeDot={{
+                      stroke: 'hsl(var(--foreground) / 0.2)',
+                      fill: 'hsl(var(--def-100))',
+                      strokeWidth: 2,
+                      r: 4,
+                    }}
+                  />
+
+                  <Line
+                    key={activeMetric.key}
+                    type="linear"
+                    dataKey={activeMetric.key}
+                    stroke={getChartColor(0)}
+                    strokeWidth={2}
+                    isAnimationActive={false}
+                    dot={
+                      data.length > 90
+                        ? false
+                        : {
+                            stroke: getChartColor(0),
+                            fill: 'hsl(var(--def-100))',
+                            strokeWidth: 1.5,
+                            r: 3,
+                          }
+                    }
+                    activeDot={{
+                      stroke: getChartColor(0),
+                      fill: 'hsl(var(--def-100))',
+                      strokeWidth: 2,
+                      r: 4,
+                    }}
+                  />
+                </LineChart>
+              </ResponsiveContainer>
+            </TooltipProvider>
+          </div>
+        </div>
+      </div>
+    </>
+  );
+}
+
+const { Tooltip, TooltipProvider } = createChartTooltip<
+  RouterOutputs['overview']['stats']['series'][number],
+  {
+    metric: (typeof TITLES)[number];
+  }
+>(({ context: { metric }, data }) => {
+  const formatDate = useFormatDateInterval('day');
+  const number = useNumber();
+
+  return (
+    <>
+      <div className="flex justify-between gap-8 text-muted-foreground">
+        <div>{formatDate(new Date(data.date))}</div>
+      </div>
+      <React.Fragment>
+        <div className="flex gap-2">
+          <div
+            className="w-[3px] rounded-full"
+            style={{ background: getChartColor(0) }}
+          />
+          <div className="col flex-1 gap-1">
+            <div className="flex items-center gap-1">{metric.title}</div>
+            <div className="flex justify-between gap-8 font-mono font-medium">
+              <div className="row gap-1">
+                {number.formatWithUnit(data[metric.key])}
+                {!!data[`prev_${metric.key}`] && (
+                  <span className="text-muted-foreground">
+                    ({number.formatWithUnit(data[`prev_${metric.key}`])})
+                  </span>
+                )}
+              </div>
+
+              <PreviousDiffIndicatorPure
+                {...getPreviousMetric(
+                  data[metric.key],
+                  data[`prev_${metric.key}`],
+                )}
+              />
+            </div>
+          </div>
+        </div>
+      </React.Fragment>
+    </>
+  );
+});

apps/dashboard/src/components/overview/overview-metrics.tsx

@@ -218,7 +218,7 @@ export default function OverviewMetrics({ projectId }: OverviewMetricsProps) {
             <OverviewLiveHistogram projectId={projectId} />
           </div>
         </div>
-        <div className="card col-span-6 p-4">
+        {/* <div className="card col-span-6 p-4">
           <ReportChart
             key={selectedMetric.id}
             options={{
@@ -233,7 +233,7 @@ export default function OverviewMetrics({ projectId }: OverviewMetricsProps) {
               lineType: 'linear',
             }}
           />
-        </div>
+        </div> */}
       </div>
     </>
   );

apps/dashboard/src/components/overview/overview-top-devices.tsx

@@ -7,11 +7,18 @@ import { useState } from 'react';
 import { NOT_SET_VALUE } from '@openpanel/constants';
 import type { IChartType } from '@openpanel/validation';
 
-import { ReportChart } from '../report-chart';
+import { useNumber } from '@/hooks/useNumerFormatter';
+import { pushModal } from '@/modals';
+import { api } from '@/trpc/client';
+import { SerieIcon } from '../report-chart/common/serie-icon';
 import { Widget, WidgetBody } from '../widget';
-import { OverviewChartToggle } from './overview-chart-toggle';
+import { OVERVIEW_COLUMNS_NAME } from './overview-constants';
 import OverviewDetailsButton from './overview-details-button';
 import { WidgetButtons, WidgetFooter, WidgetHead } from './overview-widget';
+import {
+  OverviewWidgetTableGeneric,
+  OverviewWidgetTableLoading,
+} from './overview-widget-table';
 import { useOverviewOptions } from './useOverviewOptions';
 import { useOverviewWidget } from './useOverviewWidget';
 
@@ -27,7 +34,7 @@ export default function OverviewTopDevices({
   const [chartType, setChartType] = useState<IChartType>('bar');
   const isPageFilter = filters.find((filter) => filter.name === 'path');
   const [widget, setWidget, widgets] = useOverviewWidget('tech', {
-    devices: {
+    device: {
       title: 'Top devices',
       btn: 'Devices',
       chart: {
@@ -221,7 +228,7 @@ export default function OverviewTopDevices({
         },
       },
     },
-    brands: {
+    brand: {
       title: 'Top Brands',
       btn: 'Brands',
       chart: {
@@ -257,7 +264,7 @@ export default function OverviewTopDevices({
         },
       },
     },
-    models: {
+    model: {
       title: 'Top Models',
       btn: 'Models',
       chart: {
@@ -302,11 +309,24 @@ export default function OverviewTopDevices({
     },
   });
 
+  const number = useNumber();
+
+  const query = api.overview.topGeneric.useQuery({
+    projectId,
+    interval,
+    range,
+    filters,
+    column: widget.key,
+    startDate,
+    endDate,
+  });
+
   return (
     <>
       <Widget className="col-span-6 md:col-span-3">
         <WidgetHead>
           <div className="title">{widget.title}</div>
+
           <WidgetButtons>
             {widgets.map((w) => (
               <button
@@ -321,39 +341,44 @@ export default function OverviewTopDevices({
           </WidgetButtons>
         </WidgetHead>
         <WidgetBody>
-          <ReportChart
+          {query.isLoading ? (
-            options={{
+            <OverviewWidgetTableLoading className="-m-4" />
-              ...widget.chart.options,
+          ) : (
-              hideID: true,
+            <OverviewWidgetTableGeneric
-              onClick: (item) => {
+              className="-m-4"
-                switch (widget.key) {
+              data={query.data ?? []}
-                  case 'devices':
+              column={{
-                    setFilter('device', item.names[0]);
+                name: OVERVIEW_COLUMNS_NAME[widget.key],
-                    break;
+                render(item) {
-                  case 'browser':
+                  return (
-                    setFilter('browser', item.names[0]);
+                    <div className="row items-center gap-2 min-w-0 relative">
-                    break;
+                      <SerieIcon name={item.name || NOT_SET_VALUE} />
-                  case 'browser_version':
+                      <button
-                    setFilter('browser_version', item.names[1]);
+                        type="button"
-                    break;
+                        className="truncate"
-                  case 'os':
+                        onClick={() => {
-                    setFilter('os', item.names[0]);
+                          setFilter(widget.key, item.name);
-                    break;
+                        }}
-                  case 'os_version':
+                      >
-                    setFilter('os_version', item.names[1]);
+                        {item.name || 'Not set'}
-                    break;
+                      </button>
-                }
+                    </div>
+                  );
                 },
               }}
-            report={{
-              ...widget.chart.report,
-              previous: false,
-            }}
             />
+          )}
         </WidgetBody>
         <WidgetFooter>
-          <OverviewDetailsButton chart={widget.chart.report} />
+          <OverviewDetailsButton
-          <OverviewChartToggle {...{ chartType, setChartType }} />
+            onClick={() =>
+              pushModal('OverviewTopGenericModal', {
+                projectId,
+                column: widget.key,
+              })
+            }
+          />
+          {/* <OverviewChartToggle {...{ chartType, setChartType }} /> */}
         </WidgetFooter>
       </Widget>
     </>

apps/dashboard/src/components/overview/overview-top-events/overview-top-events.tsx

@@ -9,7 +9,6 @@ import type { IChartType } from '@openpanel/validation';
 
 import { Widget, WidgetBody } from '../../widget';
 import { OverviewChartToggle } from '../overview-chart-toggle';
-import OverviewDetailsButton from '../overview-details-button';
 import { WidgetButtons, WidgetFooter, WidgetHead } from '../overview-widget';
 import { useOverviewOptions } from '../useOverviewOptions';
 import { useOverviewWidget } from '../useOverviewWidget';
@@ -175,7 +174,6 @@ export default function OverviewTopEvents({
           />
         </WidgetBody>
         <WidgetFooter>
-          <OverviewDetailsButton chart={widget.chart.report} />
           <OverviewChartToggle {...{ chartType, setChartType }} />
         </WidgetFooter>
       </Widget>

apps/dashboard/src/components/overview/overview-top-generic-modal.tsx

@@ -0,0 +1,107 @@
+'use client';
+
+import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
+
+import { ModalContent, ModalHeader } from '@/modals/Modal/Container';
+import { api } from '@/trpc/client';
+import type { IGetTopGenericInput } from '@openpanel/db';
+import { ChevronRightIcon } from 'lucide-react';
+import { useEffect } from 'react';
+import { toast } from 'sonner';
+import { SerieIcon } from '../report-chart/common/serie-icon';
+import { Button } from '../ui/button';
+import { ScrollArea } from '../ui/scroll-area';
+import {
+  OVERVIEW_COLUMNS_NAME,
+  OVERVIEW_COLUMNS_NAME_PLURAL,
+} from './overview-constants';
+import { OverviewWidgetTableGeneric } from './overview-widget-table';
+import { useOverviewOptions } from './useOverviewOptions';
+
+interface OverviewTopGenericModalProps {
+  projectId: string;
+  column: IGetTopGenericInput['column'];
+}
+
+export default function OverviewTopGenericModal({
+  projectId,
+  column,
+}: OverviewTopGenericModalProps) {
+  const [filters, setFilter] = useEventQueryFilters();
+  const { startDate, endDate, range, interval } = useOverviewOptions();
+  const query = api.overview.topGeneric.useInfiniteQuery(
+    {
+      projectId,
+      filters,
+      startDate,
+      endDate,
+      range,
+      interval,
+      limit: 50,
+      column,
+    },
+    {
+      getNextPageParam: (lastPage, pages) => {
+        if (lastPage.length === 0) {
+          return null;
+        }
+
+        return pages.length + 1;
+      },
+    },
+  );
+
+  const data = query.data?.pages.flat() || [];
+  const isEmpty = !query.hasNextPage && !query.isFetching;
+
+  const columnNamePlural = OVERVIEW_COLUMNS_NAME_PLURAL[column];
+  const columnName = OVERVIEW_COLUMNS_NAME[column];
+
+  return (
+    <ModalContent>
+      <ModalHeader title={`Top ${columnNamePlural}`} />
+      <ScrollArea className="-mx-6 px-2 max-h-[calc(80vh)]">
+        <OverviewWidgetTableGeneric
+          data={data}
+          column={{
+            name: columnName,
+            render(item) {
+              return (
+                <div className="row items-center gap-2 min-w-0 relative">
+                  <SerieIcon name={item.prefix || item.name} />
+                  <button
+                    type="button"
+                    className="truncate"
+                    onClick={() => {
+                      setFilter(column, item.name);
+                    }}
+                  >
+                    {item.prefix && (
+                      <span className="mr-1 row inline-flex items-center gap-1">
+                        <span>{item.prefix}</span>
+                        <span>
+                          <ChevronRightIcon className="size-3" />
+                        </span>
+                      </span>
+                    )}
+                    {item.name || 'Not set'}
+                  </button>
+                </div>
+              );
+            },
+          }}
+        />
+        <div className="row center-center p-4 pb-0">
+          <Button
+            variant="outline"
+            className="w-full"
+            onClick={() => query.fetchNextPage()}
+            disabled={isEmpty}
+          >
+            Load more
+          </Button>
+        </div>
+      </ScrollArea>
+    </ModalContent>
+  );
+}

apps/dashboard/src/components/overview/overview-top-geo.tsx

@@ -1,20 +1,28 @@
 'use client';
 
 import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
-import { getCountry } from '@/translations/countries';
 import { cn } from '@/utils/cn';
 import { useState } from 'react';
 
-import { NOT_SET_VALUE } from '@openpanel/constants';
 import type { IChartType } from '@openpanel/validation';
 
+import { useNumber } from '@/hooks/useNumerFormatter';
+import { pushModal } from '@/modals';
+import { api } from '@/trpc/client';
+import { NOT_SET_VALUE } from '@openpanel/constants';
+import { ChevronRightIcon } from 'lucide-react';
 import { ReportChart } from '../report-chart';
+import { SerieIcon } from '../report-chart/common/serie-icon';
 import { Widget, WidgetBody } from '../widget';
-import { OverviewChartToggle } from './overview-chart-toggle';
+import { OVERVIEW_COLUMNS_NAME } from './overview-constants';
 import OverviewDetailsButton from './overview-details-button';
 import { WidgetButtons, WidgetFooter, WidgetHead } from './overview-widget';
+import {
+  OverviewWidgetTableGeneric,
+  OverviewWidgetTableLoading,
+} from './overview-widget-table';
 import { useOverviewOptions } from './useOverviewOptions';
-import { useOverviewWidget } from './useOverviewWidget';
+import { useOverviewWidgetV2 } from './useOverviewWidget';
 
 interface OverviewTopGeoProps {
   projectId: string;
@@ -25,132 +33,31 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) {
   const [chartType, setChartType] = useState<IChartType>('bar');
   const [filters, setFilter] = useEventQueryFilters();
   const isPageFilter = filters.find((filter) => filter.name === 'path');
-  const [widget, setWidget, widgets] = useOverviewWidget('geo', {
+  const [widget, setWidget, widgets] = useOverviewWidgetV2('geo', {
-    countries: {
+    country: {
       title: 'Top countries',
       btn: 'Countries',
-      chart: {
-        options: {
-          columns: ['Country', isPageFilter ? 'Views' : 'Sessions'],
-          renderSerieName(name) {
-            return getCountry(name[0]) || NOT_SET_VALUE;
     },
-        },
+    region: {
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'country',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'Top countries',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
-    },
-    regions: {
       title: 'Top regions',
       btn: 'Regions',
-      chart: {
-        options: {
-          columns: ['Region', isPageFilter ? 'Views' : 'Sessions'],
-          renderSerieName(name) {
-            return name[1] || NOT_SET_VALUE;
     },
-        },
+    city: {
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'country',
-            },
-            {
-              id: 'B',
-              name: 'region',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'Top regions',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
-    },
-    cities: {
       title: 'Top cities',
       btn: 'Cities',
-      chart: {
-        options: {
-          columns: ['City', isPageFilter ? 'Views' : 'Sessions'],
-          renderSerieName(name) {
-            return name[1] || NOT_SET_VALUE;
     },
-        },
+  });
-        report: {
+
-          limit: 10,
+  const number = useNumber();
+
+  const query = api.overview.topGeneric.useQuery({
     projectId,
+    interval,
+    range,
+    filters,
+    column: widget.key,
     startDate,
     endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'country',
-            },
-            {
-              id: 'B',
-              name: 'city',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'Top cities',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
-    },
   });
 
   return (
@@ -158,6 +65,7 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) {
       <Widget className="col-span-6 md:col-span-3">
         <WidgetHead>
           <div className="title">{widget.title}</div>
+
           <WidgetButtons>
             {widgets.map((w) => (
               <button
@@ -172,35 +80,59 @@ export default function OverviewTopGeo({ projectId }: OverviewTopGeoProps) {
           </WidgetButtons>
         </WidgetHead>
         <WidgetBody>
-          <ReportChart
+          {query.isLoading ? (
-            options={{
+            <OverviewWidgetTableLoading className="-m-4" />
-              hideID: true,
+          ) : (
-              onClick: (item) => {
+            <OverviewWidgetTableGeneric
-                switch (widget.key) {
+              className="-m-4"
-                  case 'countries':
+              data={query.data ?? []}
-                    setWidget('regions');
+              column={{
-                    setFilter('country', item.names[0]);
+                name: OVERVIEW_COLUMNS_NAME[widget.key],
-                    break;
+                render(item) {
-                  case 'regions':
+                  return (
-                    setWidget('cities');
+                    <div className="row items-center gap-2 min-w-0 relative">
-                    setFilter('region', item.names[1]);
+                      <SerieIcon
-                    break;
+                        name={item.prefix || item.name || NOT_SET_VALUE}
-                  case 'cities':
+                      />
-                    setFilter('city', item.names[1]);
+                      <button
-                    break;
+                        type="button"
+                        className="truncate"
+                        onClick={() => {
+                          if (widget.key === 'country') {
+                            setWidget('region');
+                          } else if (widget.key === 'region') {
+                            setWidget('city');
                           }
-              },
+                          setFilter(widget.key, item.name);
-              ...widget.chart.options,
                         }}
-            report={{
+                      >
-              ...widget.chart.report,
+                        {item.prefix && (
-              previous: false,
+                          <span className="mr-1 row inline-flex items-center gap-1">
+                            <span>{item.prefix}</span>
+                            <span>
+                              <ChevronRightIcon className="size-3" />
+                            </span>
+                          </span>
+                        )}
+                        {item.name || 'Not set'}
+                      </button>
+                    </div>
+                  );
+                },
               }}
             />
+          )}
         </WidgetBody>
         <WidgetFooter>
-          <OverviewDetailsButton chart={widget.chart.report} />
+          <OverviewDetailsButton
-          <OverviewChartToggle {...{ chartType, setChartType }} />
+            onClick={() =>
+              pushModal('OverviewTopGenericModal', {
+                projectId,
+                column: widget.key,
+              })
+            }
+          />
+          {/* <OverviewChartToggle {...{ chartType, setChartType }} /> */}
         </WidgetFooter>
       </Widget>
       <Widget className="col-span-6 md:col-span-3">

apps/dashboard/src/components/overview/overview-top-pages-modal.tsx

@@ -0,0 +1,68 @@
+'use client';
+
+import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
+
+import { ModalContent, ModalHeader } from '@/modals/Modal/Container';
+import { api } from '@/trpc/client';
+import { Button } from '../ui/button';
+import { ScrollArea } from '../ui/scroll-area';
+import { OverviewWidgetTablePages } from './overview-widget-table';
+import { useOverviewOptions } from './useOverviewOptions';
+
+interface OverviewTopPagesProps {
+  projectId: string;
+}
+
+function getPath(path: string) {
+  try {
+    return new URL(path).pathname;
+  } catch {
+    return path;
+  }
+}
+
+export default function OverviewTopPagesModal({
+  projectId,
+}: OverviewTopPagesProps) {
+  const [filters, setFilter] = useEventQueryFilters();
+  const { startDate, endDate, range, interval } = useOverviewOptions();
+  const query = api.overview.topPages.useInfiniteQuery(
+    {
+      projectId,
+      filters,
+      startDate,
+      endDate,
+      mode: 'page',
+      range,
+      interval: 'day',
+      limit: 50,
+    },
+    {
+      getNextPageParam: (_, pages) => pages.length + 1,
+    },
+  );
+
+  const data = query.data?.pages.flat();
+
+  return (
+    <ModalContent>
+      <ModalHeader title="Top Pages" />
+      <ScrollArea className="-mx-6 px-2 max-h-[calc(80vh)]">
+        <OverviewWidgetTablePages
+          data={data ?? []}
+          lastColumnName={'Sessions'}
+        />
+        <div className="row center-center p-4 pb-0">
+          <Button
+            variant="outline"
+            className="w-full"
+            onClick={() => query.fetchNextPage()}
+            loading={query.isFetching}
+          >
+            Load more
+          </Button>
+        </div>
+      </ScrollArea>
+    </ModalContent>
+  );
+}

apps/dashboard/src/components/overview/overview-top-pages.tsx

@@ -2,179 +2,81 @@
 
 import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
 import { cn } from '@/utils/cn';
-import { ExternalLinkIcon, FilterIcon, Globe2Icon } from 'lucide-react';
+import { Globe2Icon } from 'lucide-react';
 import { parseAsBoolean, useQueryState } from 'nuqs';
 import { useState } from 'react';
 
-import { NOT_SET_VALUE } from '@openpanel/constants';
 import type { IChartType } from '@openpanel/validation';
 
-import { ReportChart } from '../report-chart';
+import { pushModal } from '@/modals';
+import { api } from '@/trpc/client';
 import { Button } from '../ui/button';
-import { Tooltiper } from '../ui/tooltip';
 import { Widget, WidgetBody } from '../widget';
-import { OverviewChartToggle } from './overview-chart-toggle';
 import OverviewDetailsButton from './overview-details-button';
-import OverviewTopBots from './overview-top-bots';
 import { WidgetButtons, WidgetFooter, WidgetHead } from './overview-widget';
+import {
+  OverviewWidgetTableBots,
+  OverviewWidgetTableLoading,
+  OverviewWidgetTablePages,
+} from './overview-widget-table';
 import { useOverviewOptions } from './useOverviewOptions';
-import { useOverviewWidget } from './useOverviewWidget';
+import { useOverviewWidgetV2 } from './useOverviewWidget';
 
 interface OverviewTopPagesProps {
   projectId: string;
 }
+
 export default function OverviewTopPages({ projectId }: OverviewTopPagesProps) {
   const { interval, range, previous, startDate, endDate } =
     useOverviewOptions();
-  const [chartType, setChartType] = useState<IChartType>('bar');
+  const [filters] = useEventQueryFilters();
-  const [filters, setFilter] = useEventQueryFilters();
   const [domain, setDomain] = useQueryState('d', parseAsBoolean);
-  const renderSerieName = (names: string[]) => {
+  const [widget, setWidget, widgets] = useOverviewWidgetV2('pages', {
-    if (domain) {
+    page: {
-      if (names[0] === NOT_SET_VALUE) {
-        return names[1];
-      }
-
-      return names.join('');
-    }
-    return (
-      <Tooltiper content={names.join('')} side="left" className="text-left">
-        {names[1] || NOT_SET_VALUE}
-      </Tooltiper>
-    );
-  };
-  const [widget, setWidget, widgets] = useOverviewWidget('pages', {
-    top: {
       title: 'Top pages',
       btn: 'Top pages',
-      chart: {
+      meta: {
-        options: {
+        columns: {
-          renderSerieName,
+          sessions: 'Sessions',
-          columns: ['URL', 'Views'],
-        },
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: 'screen_view',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'origin',
-            },
-            {
-              id: 'B',
-              name: 'path',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval,
-          name: 'Top pages',
-          range,
-          previous,
-          metric: 'sum',
         },
       },
     },
-    entries: {
+    entry: {
       title: 'Entry Pages',
       btn: 'Entries',
-      chart: {
+      meta: {
-        options: {
+        columns: {
-          columns: ['URL', 'Sessions'],
+          sessions: 'Entries',
-          renderSerieName,
-        },
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'origin',
-            },
-            {
-              id: 'B',
-              name: 'path',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval,
-          name: 'Entry Pages',
-          range,
-          previous,
-          metric: 'sum',
         },
       },
     },
-    exits: {
+    exit: {
       title: 'Exit Pages',
       btn: 'Exits',
-      chart: {
+      meta: {
-        options: {
+        columns: {
-          columns: ['URL', 'Sessions'],
+          sessions: 'Exits',
-          renderSerieName,
         },
-        report: {
+      },
-          limit: 10,
+    },
+    // bot: {
+    //   title: 'Bots',
+    //   btn: 'Bots',
+    // },
+  });
+
+  const query = api.overview.topPages.useQuery({
     projectId,
+    filters,
     startDate,
     endDate,
-          events: [
+    mode: widget.key,
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: 'session_end',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'origin',
-            },
-            {
-              id: 'B',
-              name: 'path',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval,
-          name: 'Exit Pages',
     range,
-          previous,
+    interval,
-          metric: 'sum',
-        },
-      },
-    },
-    bot: {
-      title: 'Bots',
-      btn: 'Bots',
-      // @ts-expect-error
-      chart: null,
-    },
   });
 
+  const data = query.data;
+
   return (
     <>
       <Widget className="col-span-6 md:col-span-3">
@@ -194,41 +96,25 @@ export default function OverviewTopPages({ projectId }: OverviewTopPagesProps) {
           </WidgetButtons>
         </WidgetHead>
         <WidgetBody>
-          {widget.key === 'bot' ? (
+          {query.isLoading ? (
-            <OverviewTopBots projectId={projectId} />
+            <OverviewWidgetTableLoading className="-m-4" />
           ) : (
-            <ReportChart
+            <>
-              options={{
+              {/*<OverviewWidgetTableBots className="-m-4" data={data ?? []} />*/}
-                hideID: true,
+              <OverviewWidgetTablePages
-                dropdownMenuContent: (serie) => [
+                className="-m-4"
-                  {
+                data={data ?? []}
-                    title: 'Visit page',
+                lastColumnName={widget.meta.columns.sessions}
-                    icon: ExternalLinkIcon,
+                showDomain={!!domain}
-                    onClick: () => {
-                      window.open(serie.names.join(''), '_blank');
-                    },
-                  },
-                  {
-                    title: 'Set filter',
-                    icon: FilterIcon,
-                    onClick: () => {
-                      setFilter('path', serie.names[1]);
-                    },
-                  },
-                ],
-                ...widget.chart.options,
-              }}
-              report={{
-                ...widget.chart.report,
-                previous: false,
-              }}
               />
+            </>
           )}
         </WidgetBody>
-        {widget.chart?.report?.name && (
         <WidgetFooter>
-            <OverviewDetailsButton chart={widget.chart.report} />
+          <OverviewDetailsButton
-            <OverviewChartToggle {...{ chartType, setChartType }} />
+            onClick={() => pushModal('OverviewTopPagesModal', { projectId })}
+          />
+          {/* <OverviewChartToggle {...{ chartType, setChartType }} /> */}
           <div className="flex-1" />
           <Button
             variant={'ghost'}
@@ -240,7 +126,6 @@ export default function OverviewTopPages({ projectId }: OverviewTopPagesProps) {
             {domain ? 'Hide domain' : 'Show domain'}
           </Button>
         </WidgetFooter>
-        )}
       </Widget>
     </>
   );

apps/dashboard/src/components/overview/overview-top-sources.tsx

@@ -2,18 +2,21 @@
 
 import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
 import { cn } from '@/utils/cn';
-import { useState } from 'react';
-
-import type { IChartType } from '@openpanel/validation';
 
+import { pushModal } from '@/modals';
+import { api } from '@/trpc/client';
 import { NOT_SET_VALUE } from '@openpanel/constants';
-import { ReportChart } from '../report-chart';
+import { SerieIcon } from '../report-chart/common/serie-icon';
 import { Widget, WidgetBody } from '../widget';
-import { OverviewChartToggle } from './overview-chart-toggle';
+import { OVERVIEW_COLUMNS_NAME } from './overview-constants';
 import OverviewDetailsButton from './overview-details-button';
 import { WidgetButtons, WidgetFooter, WidgetHead } from './overview-widget';
+import {
+  OverviewWidgetTableGeneric,
+  OverviewWidgetTableLoading,
+} from './overview-widget-table';
 import { useOverviewOptions } from './useOverviewOptions';
-import { useOverviewWidget } from './useOverviewWidget';
+import { useOverviewWidgetV2 } from './useOverviewWidget';
 
 interface OverviewTopSourcesProps {
   projectId: string;
@@ -21,300 +24,51 @@ interface OverviewTopSourcesProps {
 export default function OverviewTopSources({
   projectId,
 }: OverviewTopSourcesProps) {
-  const { interval, range, previous, startDate, endDate } =
+  const { interval, range, startDate, endDate } = useOverviewOptions();
-    useOverviewOptions();
-  const [chartType, setChartType] = useState<IChartType>('bar');
   const [filters, setFilter] = useEventQueryFilters();
-  const isPageFilter = filters.find((filter) => filter.name === 'path');
+  const [widget, setWidget, widgets] = useOverviewWidgetV2('sources', {
-  const [widget, setWidget, widgets] = useOverviewWidget('sources', {
+    referrer_name: {
-    all: {
       title: 'Top sources',
       btn: 'All',
-      chart: {
-        options: {
-          columns: ['Source', isPageFilter ? 'Views' : 'Sessions'],
     },
-        report: {
+    referrer: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters: filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'referrer_name',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'Top sources',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
-    },
-    domain: {
       title: 'Top urls',
       btn: 'URLs',
-      chart: {
-        options: {
-          columns: ['URL', isPageFilter ? 'Views' : 'Sessions'],
     },
-        report: {
+    referrer_type: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters: filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'referrer',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'Top urls',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
-    },
-    type: {
       title: 'Top types',
       btn: 'Types',
-      chart: {
-        options: {
-          columns: ['Type', isPageFilter ? 'Views' : 'Sessions'],
-        },
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters: filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'referrer_type',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'Top types',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
     },
     utm_source: {
       title: 'UTM Source',
       btn: 'Source',
-      chart: {
-        options: {
-          columns: ['Utm Source', isPageFilter ? 'Views' : 'Sessions'],
-        },
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'properties.__query.utm_source',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'UTM Source',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
     },
     utm_medium: {
       title: 'UTM Medium',
       btn: 'Medium',
-      chart: {
-        options: {
-          columns: ['Utm Medium', isPageFilter ? 'Views' : 'Sessions'],
-        },
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'properties.__query.utm_medium',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'UTM Medium',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
     },
     utm_campaign: {
       title: 'UTM Campaign',
       btn: 'Campaign',
-      chart: {
-        options: {
-          columns: ['Utm Campaign', isPageFilter ? 'Views' : 'Sessions'],
-        },
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'properties.__query.utm_campaign',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'UTM Campaign',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
     },
     utm_term: {
       title: 'UTM Term',
       btn: 'Term',
-      chart: {
-        options: {
-          columns: ['Utm Term', isPageFilter ? 'Views' : 'Sessions'],
-        },
-        report: {
-          limit: 10,
-          projectId,
-          startDate,
-          endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'properties.__query.utm_term',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'UTM Term',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
     },
     utm_content: {
       title: 'UTM Content',
       btn: 'Content',
-      chart: {
-        options: {
-          columns: ['Utm Content', isPageFilter ? 'Views' : 'Sessions'],
     },
-        report: {
+  });
-          limit: 10,
+
+  const query = api.overview.topGeneric.useQuery({
     projectId,
+    interval,
+    range,
+    filters,
+    column: widget.key,
     startDate,
     endDate,
-          events: [
-            {
-              segment: 'event',
-              filters,
-              id: 'A',
-              name: isPageFilter ? 'screen_view' : 'session_start',
-            },
-          ],
-          breakdowns: [
-            {
-              id: 'A',
-              name: 'properties.__query.utm_content',
-            },
-          ],
-          chartType,
-          lineType: 'monotone',
-          interval: interval,
-          name: 'UTM Content',
-          range: range,
-          previous: previous,
-          metric: 'sum',
-        },
-      },
-    },
   });
 
   return (
@@ -337,51 +91,53 @@ export default function OverviewTopSources({
           </WidgetButtons>
         </WidgetHead>
         <WidgetBody>
-          <ReportChart
+          {query.isLoading ? (
-            report={{
+            <OverviewWidgetTableLoading className="-m-4" />
-              ...widget.chart.report,
+          ) : (
-              previous: false,
+            <OverviewWidgetTableGeneric
-            }}
+              className="-m-4"
-            options={{
+              data={query.data ?? []}
-              ...widget.chart.options,
+              column={{
-              renderSerieName: (name) =>
+                name: OVERVIEW_COLUMNS_NAME[widget.key],
-                name[0] === NOT_SET_VALUE ? 'Direct / Not set' : name[0],
+                render(item) {
-              onClick: (item) => {
+                  return (
-                switch (widget.key) {
+                    <div className="row items-center gap-2 min-w-0 relative">
-                  case 'all':
+                      <SerieIcon name={item.name || NOT_SET_VALUE} />
-                    setFilter('referrer_name', item.names[0]);
+                      <button
-                    setWidget('domain');
+                        type="button"
-                    break;
+                        className="truncate"
-                  case 'domain':
+                        onClick={() => {
-                    setFilter('referrer', item.names[0]);
+                          if (widget.key.startsWith('utm_')) {
-                    break;
+                            setFilter(
-                  case 'type':
+                              `properties.__query.${widget.key}`,
-                    setFilter('referrer_type', item.names[0]);
+                              item.name,
-                    setWidget('domain');
+                            );
-                    break;
+                          } else {
-                  case 'utm_source':
+                            setFilter(widget.key, item.name);
-                    setFilter('properties.__query.utm_source', item.names[0]);
-                    break;
-                  case 'utm_medium':
-                    setFilter('properties.__query.utm_medium', item.names[0]);
-                    break;
-                  case 'utm_campaign':
-                    setFilter('properties.__query.utm_campaign', item.names[0]);
-                    break;
-                  case 'utm_term':
-                    setFilter('properties.__query.utm_term', item.names[0]);
-                    break;
-                  case 'utm_content':
-                    setFilter('properties.__query.utm_content', item.names[0]);
-                    break;
                           }
+                        }}
+                      >
+                        {(item.name || 'Direct / Not set')
+                          .replace(/https?:\/\//, '')
+                          .replace('www.', '')}
+                      </button>
+                    </div>
+                  );
                 },
               }}
             />
+          )}
         </WidgetBody>
         <WidgetFooter>
-          <OverviewDetailsButton chart={widget.chart.report} />
+          <OverviewDetailsButton
-          <OverviewChartToggle {...{ chartType, setChartType }} />
+            onClick={() =>
+              pushModal('OverviewTopGenericModal', {
+                projectId,
+                column: widget.key,
+              })
+            }
+          />
+          {/* <OverviewChartToggle {...{ chartType, setChartType }} /> */}
         </WidgetFooter>
       </Widget>
     </>

apps/dashboard/src/components/overview/overview-widget-table.tsx

@@ -0,0 +1,330 @@
+import { useEventQueryFilters } from '@/hooks/useEventQueryFilters';
+import { useNumber } from '@/hooks/useNumerFormatter';
+import type { RouterOutputs } from '@/trpc/client';
+import { cn } from '@/utils/cn';
+import { ExternalLinkIcon } from 'lucide-react';
+import { SerieIcon } from '../report-chart/common/serie-icon';
+import { Skeleton } from '../skeleton';
+import { Tooltiper } from '../ui/tooltip';
+import { WidgetTable, type Props as WidgetTableProps } from '../widget-table';
+
+type Props<T> = WidgetTableProps<T> & {
+  getColumnPercentage: (item: T) => number;
+};
+
+export const OverviewWidgetTable = <T,>({
+  data,
+  keyExtractor,
+  columns,
+  getColumnPercentage,
+  className,
+}: Props<T>) => {
+  return (
+    <div className={cn(className)}>
+      <WidgetTable
+        data={data ?? []}
+        keyExtractor={keyExtractor}
+        className={'text-sm min-h-[358px] @container'}
+        columnClassName="px-2 group/row items-center"
+        eachRow={(item) => {
+          return (
+            <div className="absolute inset-0 !p-0">
+              <div
+                className="h-full bg-def-200 group-hover/row:bg-blue-200 dark:group-hover/row:bg-blue-900 transition-colors relative"
+                style={{
+                  width: `${getColumnPercentage(item) * 100}%`,
+                }}
+              />
+            </div>
+          );
+        }}
+        columns={columns.map((column, index) => {
+          return {
+            ...column,
+            className: cn(
+              index === 0
+                ? 'w-full flex-1 font-medium min-w-0'
+                : 'text-right justify-end row w-20 font-mono',
+              index !== 0 &&
+                index !== columns.length - 1 &&
+                'hidden @[310px]:row',
+              column.className,
+            ),
+          };
+        })}
+      />
+    </div>
+  );
+};
+
+export function OverviewWidgetTableLoading({
+  className,
+}: {
+  className?: string;
+}) {
+  return (
+    <OverviewWidgetTable
+      className={className}
+      data={[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}
+      keyExtractor={(item) => item.toString()}
+      getColumnPercentage={() => 0}
+      columns={[
+        {
+          name: 'Path',
+          render: () => <Skeleton className="h-4 w-1/3" />,
+        },
+        {
+          name: 'BR',
+          render: () => <Skeleton className="h-4 w-[30px]" />,
+        },
+        // {
+        //   name: 'Duration',
+        //   render: () => <Skeleton className="h-4 w-[30px]" />,
+        // },
+        {
+          name: 'Sessions',
+          render: () => <Skeleton className="h-4 w-[30px]" />,
+        },
+      ]}
+    />
+  );
+}
+
+function getPath(path: string, showDomain = false) {
+  try {
+    const url = new URL(path);
+    if (showDomain) {
+      return url.hostname + url.pathname;
+    }
+    return url.pathname;
+  } catch {
+    return path;
+  }
+}
+
+export function OverviewWidgetTablePages({
+  data,
+  lastColumnName,
+  className,
+  showDomain = false,
+}: {
+  className?: string;
+  lastColumnName: string;
+  data: {
+    origin: string;
+    path: string;
+    avg_duration: number;
+    bounce_rate: number;
+    sessions: number;
+  }[];
+  showDomain?: boolean;
+}) {
+  const [filters, setFilter] = useEventQueryFilters();
+  const number = useNumber();
+  const maxSessions = Math.max(...data.map((item) => item.sessions));
+  return (
+    <OverviewWidgetTable
+      className={className}
+      data={data ?? []}
+      keyExtractor={(item) => item.path + item.origin}
+      getColumnPercentage={(item) => item.sessions / maxSessions}
+      columns={[
+        {
+          name: 'Path',
+          render(item) {
+            return (
+              <Tooltiper asChild content={item.origin + item.path} side="left">
+                <div className="row items-center gap-2 min-w-0 relative">
+                  <SerieIcon name={item.origin} />
+                  <button
+                    type="button"
+                    className="truncate"
+                    onClick={() => {
+                      setFilter('path', item.path);
+                      setFilter('origin', item.origin);
+                    }}
+                  >
+                    {showDomain ? (
+                      <>
+                        <span className="opacity-40">{item.origin}</span>
+                        <span>{item.path}</span>
+                      </>
+                    ) : (
+                      item.path
+                    )}
+                  </button>
+                  <a
+                    href={item.origin + item.path}
+                    target="_blank"
+                    rel="noreferrer"
+                  >
+                    <ExternalLinkIcon className="size-3 group-hover/row:opacity-100 opacity-0 transition-opacity" />
+                  </a>
+                </div>
+              </Tooltiper>
+            );
+          },
+        },
+        {
+          name: 'BR',
+          className: 'w-16',
+          render(item) {
+            return number.shortWithUnit(item.bounce_rate, '%');
+          },
+        },
+        {
+          name: 'Duration',
+          render(item) {
+            return number.shortWithUnit(item.avg_duration, 'min');
+          },
+        },
+        {
+          name: lastColumnName,
+          // className: 'w-28',
+          render(item) {
+            return (
+              <div className="row gap-2 justify-end">
+                <span className="font-semibold">
+                  {number.short(item.sessions)}
+                </span>
+              </div>
+            );
+          },
+        },
+      ]}
+    />
+  );
+}
+
+export function OverviewWidgetTableBots({
+  data,
+  className,
+}: {
+  className?: string;
+  data: {
+    total_sessions: number;
+    origin: string;
+    path: string;
+    sessions: number;
+    avg_duration: number;
+    bounce_rate: number;
+  }[];
+}) {
+  const [filters, setFilter] = useEventQueryFilters();
+  const number = useNumber();
+  const maxSessions = Math.max(...data.map((item) => item.sessions));
+  return (
+    <OverviewWidgetTable
+      className={className}
+      data={data ?? []}
+      keyExtractor={(item) => item.path + item.origin}
+      getColumnPercentage={(item) => item.sessions / maxSessions}
+      columns={[
+        {
+          name: 'Path',
+          render(item) {
+            return (
+              <Tooltiper asChild content={item.origin + item.path} side="left">
+                <div className="row items-center gap-2 min-w-0 relative">
+                  <SerieIcon name={item.origin} />
+                  <button
+                    type="button"
+                    className="truncate"
+                    onClick={() => {
+                      setFilter('path', item.path);
+                    }}
+                  >
+                    {getPath(item.path)}
+                  </button>
+                  <a
+                    href={item.origin + item.path}
+                    target="_blank"
+                    rel="noreferrer"
+                  >
+                    <ExternalLinkIcon className="size-3 group-hover/row:opacity-100 opacity-0 transition-opacity" />
+                  </a>
+                </div>
+              </Tooltiper>
+            );
+          },
+        },
+        {
+          name: 'Bot',
+          // className: 'w-28',
+          render(item) {
+            return (
+              <div className="row gap-2 justify-end">
+                <span className="font-semibold">Google bot</span>
+              </div>
+            );
+          },
+        },
+        {
+          name: 'Date',
+          // className: 'w-28',
+          render(item) {
+            return (
+              <div className="row gap-2 justify-end">
+                <span className="font-semibold">Google bot</span>
+              </div>
+            );
+          },
+        },
+      ]}
+    />
+  );
+}
+
+export function OverviewWidgetTableGeneric({
+  data,
+  column,
+  className,
+}: {
+  className?: string;
+  data: RouterOutputs['overview']['topGeneric'];
+  column: {
+    name: string;
+    render: (
+      item: RouterOutputs['overview']['topGeneric'][number],
+    ) => React.ReactNode;
+  };
+}) {
+  const number = useNumber();
+  const maxSessions = Math.max(...data.map((item) => item.sessions));
+  return (
+    <OverviewWidgetTable
+      className={className}
+      data={data ?? []}
+      keyExtractor={(item) => item.name}
+      getColumnPercentage={(item) => item.sessions / maxSessions}
+      columns={[
+        column,
+        {
+          name: 'BR',
+          className: 'w-16',
+          render(item) {
+            return number.shortWithUnit(item.bounce_rate, '%');
+          },
+        },
+        // {
+        //   name: 'Duration',
+        //   render(item) {
+        //     return number.shortWithUnit(item.avg_session_duration, 'min');
+        //   },
+        // },
+        {
+          name: 'Sessions',
+          render(item) {
+            return (
+              <div className="row gap-2 justify-end">
+                <span className="font-semibold">
+                  {number.short(item.sessions)}
+                </span>
+              </div>
+            );
+          },
+        },
+      ]}
+    />
+  );
+}

apps/dashboard/src/components/overview/useOverviewOptions.ts

@@ -78,6 +78,7 @@ export function useOverviewOptions() {
         setStartDate(null);
         setEndDate(null);
         setStorageItem('range', value);
+        setInterval(null);
       }
       setRange(value);
     },

apps/dashboard/src/components/overview/useOverviewWidget.tsx

@@ -30,3 +30,27 @@ export function useOverviewWidget<T extends string>(
     })),
   ] as const;
 }
+
+export function useOverviewWidgetV2<T extends string>(
+  key: string,
+  widgets: Record<T, { title: string; btn: string; meta?: any }>,
+) {
+  const keys = Object.keys(widgets) as T[];
+  const [widget, setWidget] = useQueryState<T>(
+    key,
+    parseAsStringEnum(keys)
+      .withDefault(keys[0]!)
+      .withOptions({ history: 'push' }),
+  );
+  return [
+    {
+      ...widgets[widget],
+      key: widget,
+    },
+    setWidget,
+    mapKeys(widgets).map((key) => ({
+      ...widgets[key],
+      key,
+    })),
+  ] as const;
+}

apps/dashboard/src/components/projects/project-card.tsx

@@ -52,7 +52,7 @@ function ProjectCard({ id, domain, name, organizationId }: IServiceProject) {
 
 async function ProjectChart({ id }: { id: string }) {
   const chart = await chQuery<{ value: number; date: string }>(
-    `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`,
+    `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.sessions} WHERE sign = 1 AND project_id = ${escape(id)} AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`,
   );
 
   return (
@@ -73,27 +73,27 @@ function Metric({ value, label }: { value: React.ReactNode; label: string }) {
 
 async function ProjectMetrics({ id }: { id: string }) {
   const [metrics] = await chQuery<{
-    total: number;
+    months_3: number;
     month: number;
     day: number;
   }>(
     `
       SELECT
       (
-        SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)}
+        SELECT uniq(DISTINCT profile_id) as count FROM ${TABLE_NAMES.sessions} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '6 months'
-      ) as total, 
+      ) as months_3, 
       (
-        SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month'
+        SELECT uniq(DISTINCT profile_id) as count FROM ${TABLE_NAMES.sessions} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month'
       ) as month,
       (
-        SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day'
+        SELECT uniq(DISTINCT profile_id) as count FROM ${TABLE_NAMES.sessions} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day'
       ) as day
     `,
   );
 
   return (
     <FadeIn className="flex gap-4">
-      <Metric label="Total" value={shortNumber('en')(metrics?.total)} />
+      <Metric label="3 months" value={shortNumber('en')(metrics?.months_3)} />
       <Metric label="Month" value={shortNumber('en')(metrics?.month)} />
       <Metric label="24h" value={shortNumber('en')(metrics?.day)} />
     </FadeIn>

apps/dashboard/src/components/report-chart/common/previous-diff-indicator.tsx

@@ -91,3 +91,65 @@ export function PreviousDiffIndicator({
     </>
   );
 }
+
+interface PreviousDiffIndicatorPureProps {
+  diff?: number | null | undefined;
+  state?: string | null | undefined;
+  inverted?: boolean;
+  size?: 'sm' | 'lg' | 'md';
+  className?: string;
+  showPrevious?: boolean;
+}
+
+export function PreviousDiffIndicatorPure({
+  diff,
+  state,
+  inverted,
+  size = 'sm',
+  className,
+  showPrevious = true,
+}: PreviousDiffIndicatorPureProps) {
+  const variant = getDiffIndicator(
+    inverted,
+    state,
+    'bg-emerald-300',
+    'bg-rose-300',
+    undefined,
+  );
+
+  if (diff === null || diff === undefined || !showPrevious) {
+    return null;
+  }
+
+  const renderIcon = () => {
+    if (state === 'positive') {
+      return <ArrowUpIcon strokeWidth={3} size={10} color="#000" />;
+    }
+    if (state === 'negative') {
+      return <ArrowDownIcon strokeWidth={3} size={10} color="#000" />;
+    }
+    return null;
+  };
+
+  return (
+    <div
+      className={cn(
+        'flex items-center gap-1 font-mono font-medium',
+        size === 'lg' && 'gap-2',
+        className,
+      )}
+    >
+      <div
+        className={cn(
+          'flex size-2.5 items-center justify-center rounded-full',
+          variant,
+          size === 'lg' && 'size-8',
+          size === 'md' && 'size-6',
+        )}
+      >
+        {renderIcon()}
+      </div>
+      {diff.toFixed(1)}%
+    </div>
+  );
+}

apps/dashboard/src/components/report-chart/common/serie-icon.urls.ts

@@ -21,7 +21,7 @@ const data = {
   'samsung internet': 'https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Samsung_Internet_logo.svg/1024px-Samsung_Internet_logo.svg.png',
   'vstat.info': 'https://vstat.info',
   'yahoo!': 'https://yahoo.com',
-  android: 'https://image.similarpng.com/very-thumbnail/2020/08/Android-icon-on-transparent--background-PNG.png',
+  android: 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/d7/Android_robot.svg/1745px-Android_robot.svg.png',
   'android browser': 'https://image.similarpng.com/very-thumbnail/2020/08/Android-icon-on-transparent--background-PNG.png',
   silk: 'https://m.media-amazon.com/images/I/51VCjQCvF0L.png',
   kakaotalk: 'https://www.kakaocorp.com/',

apps/dashboard/src/components/report-chart/funnel/chart.tsx

@@ -11,6 +11,7 @@ import { last } from 'ramda';
 import { getPreviousMetric, round } from '@openpanel/common';
 import { alphabetIds } from '@openpanel/constants';
 
+import { useNumber } from '@/hooks/useNumerFormatter';
 import { PreviousDiffIndicator } from '../common/previous-diff-indicator';
 import { useReportChartContext } from '../context';
 import { MetricCardNumber } from '../metric/metric-card';
@@ -36,6 +37,7 @@ export function Chart({
     previous,
   },
 }: Props) {
+  const number = useNumber();
   const { isEditMode } = useReportChartContext();
   const mostDropoffs = findMostDropoffs(steps);
   const lastStep = last(steps)!;
@@ -50,8 +52,8 @@ export function Chart({
         )}
       >
         <div className="flex items-center gap-8 p-4 px-8">
-          <div className="flex flex-1 items-center gap-8 min-w-0">
           <MetricCardNumber
+            className="flex-1"
             label="Converted"
             value={lastStep.count}
             enhancer={
@@ -62,6 +64,7 @@ export function Chart({
             }
           />
           <MetricCardNumber
+            className="flex-1"
             label="Percent"
             value={`${totalSessions ? round((lastStep.count / totalSessions) * 100, 2) : 0}%`}
             enhancer={
@@ -84,7 +87,6 @@ export function Chart({
           />
         </div>
       </div>
-      </div>
       <div className="col divide-y divide-def-200">
         {steps.map((step, index) => {
           const percent = (step.count / totalSessions) * 100;
@@ -109,7 +111,9 @@ export function Chart({
                         <span>
                           Last period:{' '}
                           <span className="font-mono">
-                            {previous?.steps?.[index]?.previousCount}
+                            {number.format(
+                              previous?.steps?.[index]?.previousCount,
+                            )}
                           </span>
                         </span>
                         <PreviousDiffIndicator
@@ -127,7 +131,7 @@ export function Chart({
                       </span>
                       <div className="flex items-center gap-4">
                         <span className="text-lg font-mono">
-                          {step.previousCount}
+                          {number.format(step.previousCount)}
                         </span>
                       </div>
                     </div>
@@ -139,7 +143,9 @@ export function Chart({
                         <span>
                           Last period:{' '}
                           <span className="font-mono">
-                            {previous?.steps?.[index]?.dropoffCount}
+                            {number.format(
+                              previous?.steps?.[index]?.dropoffCount,
+                            )}
                           </span>
                         </span>
                         <PreviousDiffIndicator
@@ -164,7 +170,7 @@ export function Chart({
                           )}
                         >
                           {isMostDropoffs && <AlertCircleIcon size={14} />}
-                          {step.dropoffCount}
+                          {number.format(step.dropoffCount)}
                         </span>
                       </div>
                     </div>
@@ -176,7 +182,7 @@ export function Chart({
                         <span>
                           Last period:{' '}
                           <span className="font-mono">
-                            {previous?.steps?.[index]?.count}
+                            {number.format(previous?.steps?.[index]?.count)}
                           </span>
                         </span>
                         <PreviousDiffIndicator
@@ -193,7 +199,9 @@ export function Chart({
                         Current:
                       </span>
                       <div className="flex items-center gap-4">
-                        <span className="text-lg font-mono">{step.count}</span>
+                        <span className="text-lg font-mono">
+                          {number.format(step.count)}
+                        </span>
                       </div>
                     </div>
                   </TooltipComplete>
@@ -204,7 +212,7 @@ export function Chart({
                         <span>
                           Last period:{' '}
                           <span className="font-mono">
-                            {previous?.steps?.[index]?.count}
+                            {number.format(previous?.steps?.[index]?.count)}
                           </span>
                         </span>
                         <PreviousDiffIndicator

apps/dashboard/src/components/widget-table.tsx

@@ -1,6 +1,6 @@
 import { cn } from '@/utils/cn';
 
-interface Props<T> {
+export interface Props<T> {
   columns: {
     name: string;
     render: (item: T) => React.ReactNode;
@@ -9,6 +9,8 @@ interface Props<T> {
   keyExtractor: (item: T) => string;
   data: T[];
   className?: string;
+  eachRow?: (item: T) => React.ReactNode;
+  columnClassName?: string;
 }
 
 export const WidgetTableHead = ({
@@ -21,7 +23,7 @@ export const WidgetTableHead = ({
   return (
     <thead
       className={cn(
-        'text-def-1000 sticky top-0 z-10 border-b border-border bg-def-100  [&_th:last-child]:text-right [&_th]:whitespace-nowrap [&_th]:p-4 [&_th]:py-2 [&_th]:text-left [&_th]:font-medium',
+        'text-def-1000 sticky top-0 z-10 border-b border-border bg-def-100  [&_th:last-child]:text-right [&_th]:whitespace-nowrap [&_th]:p-4 [&_th]:py-2 [&_th]:text-right [&_th:first-child]:text-left [&_th]:font-medium',
         className,
       )}
     >
@@ -35,36 +37,69 @@ export function WidgetTable<T>({
   columns,
   data,
   keyExtractor,
+  eachRow,
+  columnClassName,
 }: Props<T>) {
   return (
     <div className="w-full overflow-x-auto">
-      <table className={cn('w-full', className)}>
+      <div className={cn('w-full', className)}>
-        <WidgetTableHead>
+        <div
-          <tr>
+          className={cn(
-            {columns.map((column) => (
+            'border-b border-border text-right last:border-0 [&_div:first-child]:text-left grid',
-              <th key={column.name} className={cn(column.className)}>
+            '[&>div]:p-2',
-                {column.name}
+            columnClassName,
-              </th>
+          )}
-            ))}
+          style={{
-          </tr>
+            gridTemplateColumns:
-        </WidgetTableHead>
+              columns.length > 1
-        <tbody>
+                ? `1fr ${columns
-          {data.map((item) => (
+                    .slice(1)
-            <tr
+                    .map((col) => 'auto')
-              key={keyExtractor(item)}
+                    .join(' ')}`
-              className={
+                : '1fr',
-                'border-b border-border text-right last:border-0 [&_td:first-child]:text-left [&_td]:p-4'
+          }}
-              }
         >
           {columns.map((column) => (
-                <td key={column.name} className={cn(column.className)}>
+            <div
+              key={column.name}
+              className={cn(column.className, 'font-medium font-sans text-sm')}
+            >
+              {column.name}
+            </div>
+          ))}
+        </div>
+        <div className="col">
+          {data.map((item) => (
+            <div
+              key={keyExtractor(item)}
+              className={cn(
+                'border-b border-border text-right last:border-0 [&_div:first-child]:text-left grid relative',
+                '[&>div]:p-2',
+                columnClassName,
+              )}
+              style={{
+                gridTemplateColumns:
+                  columns.length > 1
+                    ? `1fr ${columns
+                        .slice(1)
+                        .map((col) => 'auto')
+                        .join(' ')}`
+                    : '1fr',
+              }}
+            >
+              {eachRow?.(item)}
+              {columns.map((column) => (
+                <div
+                  key={column.name}
+                  className={cn(column.className, 'relative h-8')}
+                >
                   {column.render(item)}
-                </td>
+                </div>
               ))}
-            </tr>
+            </div>
           ))}
-        </tbody>
+        </div>
-      </table>
+      </div>
     </div>
   );
 }

apps/dashboard/src/hooks/useEventQueryFilters.ts

@@ -80,7 +80,7 @@ export function useEventQueryFilters(options: NuqsOptions = {}) {
             if (filter.name === name) {
               return {
                 ...filter,
-                operator,
+                operator: newValue.length === 0 ? 'isNull' : operator,
                 value: newValue,
               };
             }
@@ -93,7 +93,7 @@ export function useEventQueryFilters(options: NuqsOptions = {}) {
           {
             id: name,
             name,
-            operator,
+            operator: newValue.length === 0 ? 'isNull' : operator,
             value: newValue,
           },
         ];
@@ -102,7 +102,14 @@ export function useEventQueryFilters(options: NuqsOptions = {}) {
     [setFilters],
   );
 
-  return [filters, setFilter, setFilters] as const;
+  const removeFilter = useCallback(
+    (name: string) => {
+      setFilters((prev) => prev.filter((filter) => filter.name !== name));
+    },
+    [setFilters],
+  );
+
+  return [filters, setFilter, setFilters, removeFilter] as const;
 }
 
 export const eventQueryNamesFilter = parseAsArrayOf(parseAsString).withDefault(

apps/dashboard/src/hooks/useNumerFormatter.ts

@@ -66,6 +66,9 @@ export function useNumber() {
       if (unit === 'min') {
         return fancyMinutes(value);
       }
+      if (unit === '%') {
+        return `${format(round(value * 100, 1))}${unit ? ` ${unit}` : ''}`;
+      }
       return `${format(value)}${unit ? ` ${unit}` : ''}`;
     },
   };

apps/dashboard/src/modals/Modal/Container.tsx

@@ -32,7 +32,7 @@ export function ModalHeader({
   return (
     <div
       className={cn(
-        'relative -m-6 mb-6 flex justify-between rounded-t-lg bg-gradient-to-b from-def-300 to-background p-6 pb-0',
+        'relative -m-6 mb-4 flex justify-between rounded-t-lg bg-gradient-to-b from-def-300 to-background p-6 pb-0',
         className,
       )}
       style={{}}

apps/dashboard/src/modals/event-details.tsx

@@ -13,13 +13,14 @@ import { ModalContent, ModalHeader } from './Modal/Container';
 
 interface Props {
   id: string;
+  createdAt?: Date;
+  projectId: string;
 }
 
-export default function EventDetails({ id }: Props) {
+export default function EventDetails({ id, createdAt, projectId }: Props) {
-  const { projectId } = useAppParams();
   const [, setEvents] = useEventQueryNamesFilter();
   const [, setFilter] = useEventQueryFilters();
-  const query = api.event.byId.useQuery({ id, projectId });
+  const query = api.event.byId.useQuery({ id, projectId, createdAt });
 
   if (query.isLoading || query.isFetching) {
     return null;

apps/dashboard/src/modals/index.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { Loader } from 'lucide-react';
+import { Loader2Icon } from 'lucide-react';
 import dynamic from 'next/dynamic';
 import { createPushModal } from 'pushmodal';
 
@@ -9,11 +9,23 @@ import { ModalContent } from './Modal/Container';
 
 const Loading = () => (
   <ModalContent className="flex items-center justify-center p-16">
-    <Loader className="animate-spin" size={40} />
+    <Loader2Icon className="animate-spin" size={40} />
   </ModalContent>
 );
 
 const modals = {
+  OverviewTopPagesModal: dynamic(
+    () => import('../components/overview/overview-top-pages-modal'),
+    {
+      loading: Loading,
+    },
+  ),
+  OverviewTopGenericModal: dynamic(
+    () => import('../components/overview/overview-top-generic-modal'),
+    {
+      loading: Loading,
+    },
+  ),
   RequestPasswordReset: dynamic(() => import('./request-reset-password'), {
     loading: Loading,
   }),

apps/dashboard/src/utils/getters.ts

@@ -8,16 +8,16 @@ export function getProfileName(
     return '';
   }
 
-  if (!profile.isExternal) {
+  const name =
-    if (short) {
+    [profile.firstName, profile.lastName].filter(Boolean).join(' ') ||
+    profile.email;
+
+  if (!name) {
+    if (short && profile.id.length > 10) {
       return `${profile.id.slice(0, 4)}...${profile.id.slice(-4)}`;
     }
     return profile.id;
   }
 
-  return (
+  return name;
-    [profile.firstName, profile.lastName].filter(Boolean).join(' ') ||
-    profile.email ||
-    profile.id
-  );
 }

apps/worker/src/boot-cron.ts

@@ -29,6 +29,11 @@ export async function bootCron() {
       type: 'flushProfiles',
       pattern: 1000 * 60,
     },
+    {
+      name: 'flush',
+      type: 'flushSessions',
+      pattern: 1000 * 10,
+    },
   ];
 
   if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {

apps/worker/src/jobs/cron.delete-projects.ts

@@ -26,6 +26,9 @@ export async function deleteProjects() {
 
   await ch.command({
     query: `DELETE FROM ${TABLE_NAMES.events} WHERE project_id IN (${projects.map((project) => escape(project.id)).join(',')});`,
+    clickhouse_settings: {
+      lightweight_deletes_sync: 0,
+    },
   });
 
   logger.info(`Deleted ${projects.length} projects`, {

apps/worker/src/jobs/cron.ts

@@ -1,6 +1,6 @@
 import type { Job } from 'bullmq';
 
-import { eventBuffer, profileBuffer } from '@openpanel/db';
+import { eventBuffer, profileBuffer, sessionBuffer } from '@openpanel/db';
 import type { CronQueuePayload } from '@openpanel/queue';
 
 import { deleteProjects } from './cron.delete-projects';
@@ -18,6 +18,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
     case 'flushProfiles': {
       return await profileBuffer.tryFlush();
     }
+    case 'flushSessions': {
+      return await sessionBuffer.tryFlush();
+    }
     case 'ping': {
       return await ping();
     }

apps/worker/src/metrics.ts

@@ -1,6 +1,12 @@
 import client from 'prom-client';
 
-import { botBuffer, db, eventBuffer, profileBuffer } from '@openpanel/db';
+import {
+  botBuffer,
+  db,
+  eventBuffer,
+  profileBuffer,
+  sessionBuffer,
+} from '@openpanel/db';
 import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';
 
 const Registry = client.Registry;
@@ -98,3 +104,14 @@ register.registerMetric(
     },
   }),
 );
+
+register.registerMetric(
+  new client.Gauge({
+    name: `buffer_${sessionBuffer.name}_count`,
+    help: 'Number of unprocessed sessions',
+    async collect() {
+      const metric = await sessionBuffer.getBufferSize();
+      this.set(metric);
+    },
+  }),
+);

apps/worker/src/utils/session-handler.ts

@@ -77,6 +77,21 @@ export async function getSessionEnd({
       throw new Error('Invalid session end job');
     }
 
+    // If the profile_id is set and it's different from the device_id, we need to update the profile_id
+    if (
+      sessionEnd.job.data.payload.profileId !== profileId &&
+      sessionEnd.job.data.payload.profileId ===
+        sessionEnd.job.data.payload.deviceId
+    ) {
+      await sessionEnd.job.updateData({
+        ...sessionEnd.job.data,
+        payload: {
+          ...sessionEnd.job.data.payload,
+          profileId,
+        },
+      });
+    }
+
     await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
   }
 

docker-compose.yml

@@ -28,7 +28,7 @@ services:
       - 8080:8080
 
   op-ch:
-    image: clickhouse/clickhouse-server:24.3.2-alpine
+    image: clickhouse/clickhouse-server:24.12.2.29-alpine
     restart: always
     volumes:
       - ./docker/data/op-ch-data:/var/lib/clickhouse

package.json

@@ -30,6 +30,9 @@
     "typescript": "^5.2.2",
     "winston": "^3.14.2"
   },
+  "resolutions": {
+    "zod": "3.22.4"
+  },
   "trustedDependencies": [
     "@biomejs/biome",
     "@prisma/client",

packages/constants/index.ts

@@ -70,6 +70,8 @@ export const operators = {
   startsWith: 'Starts with',
   endsWith: 'Ends with',
   regex: 'Regex',
+  isNull: 'Is null',
+  isNotNull: 'Is not null',
 } as const;
 
 export const chartTypes = {

packages/db/code-migrations/3-init-ch.sql

@@ -0,0 +1,167 @@
+CREATE DATABASE IF NOT EXISTS openpanel;
+
+---
+
+CREATE TABLE IF NOT EXISTS self_hosting (
+  `created_at` Date,
+  `domain` String,
+  `count` UInt64
+)
+ENGINE = MergeTree()
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (domain, created_at);
+
+---
+
+CREATE TABLE IF NOT EXISTS events (
+  `id` UUID DEFAULT generateUUIDv4(),
+  `name` LowCardinality(String),
+  `sdk_name` LowCardinality(String),
+  `sdk_version` LowCardinality(String),
+  `device_id` String CODEC(ZSTD(3)),
+  `profile_id` String CODEC(ZSTD(3)),
+  `project_id` String CODEC(ZSTD(3)),
+  `session_id` String CODEC(LZ4),
+  `path` String CODEC(ZSTD(3)),
+  `origin` String CODEC(ZSTD(3)),
+  `referrer` String CODEC(ZSTD(3)),
+  `referrer_name` String CODEC(ZSTD(3)),
+  `referrer_type` LowCardinality(String),
+  `duration` UInt64 CODEC(Delta(4), LZ4),
+  `properties` Map(String, String) CODEC(ZSTD(3)),
+  `created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),
+  `country` LowCardinality(FixedString(2)),
+  `city` String,
+  `region` LowCardinality(String),
+  `longitude` Nullable(Float32) CODEC(Gorilla, LZ4),
+  `latitude` Nullable(Float32) CODEC(Gorilla, LZ4),
+  `os` LowCardinality(String),
+  `os_version` LowCardinality(String),
+  `browser` LowCardinality(String),
+  `browser_version` LowCardinality(String),
+  `device` LowCardinality(String),
+  `brand` LowCardinality(String),
+  `model` LowCardinality(String),
+  `imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),
+  INDEX idx_name name TYPE bloom_filter GRANULARITY 1,
+  INDEX idx_properties_bounce properties['__bounce'] TYPE set(3) GRANULARITY 1,
+  INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1,
+  INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree()
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (project_id, toDate(created_at), profile_id, name)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE TABLE IF NOT EXISTS events_bots (
+  `id` UUID DEFAULT generateUUIDv4(),
+  `project_id` String,
+  `name` String,
+  `type` String,
+  `path` String,
+  `created_at` DateTime64(3)
+)
+ENGINE = MergeTree()
+ORDER BY (project_id, created_at)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE TABLE IF NOT EXISTS profiles (
+  `id` String CODEC(ZSTD(3)),
+  `is_external` Bool,
+  `first_name` String CODEC(ZSTD(3)),
+  `last_name` String CODEC(ZSTD(3)),
+  `email` String CODEC(ZSTD(3)),
+  `avatar` String CODEC(ZSTD(3)),
+  `properties` Map(String, String) CODEC(ZSTD(3)),
+  `project_id` String CODEC(ZSTD(3)),
+  `created_at` DateTime64(3) CODEC(Delta(4), LZ4),
+  INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
+  INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
+  INDEX idx_email email TYPE bloom_filter GRANULARITY 1
+)
+ENGINE = ReplacingMergeTree(created_at)
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (project_id, id)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE TABLE IF NOT EXISTS profile_aliases (
+  `project_id` String,
+  `profile_id` String,
+  `alias` String,
+  `created_at` DateTime
+)
+ENGINE = MergeTree()
+ORDER BY (project_id, profile_id, alias, created_at)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv
+ENGINE = AggregatingMergeTree()
+PARTITION BY toYYYYMMDD(date)
+ORDER BY (project_id, date)
+AS SELECT
+        toDate(created_at) as date,
+        uniqState(profile_id) as profile_id,
+        project_id
+      FROM events
+      GROUP BY date, project_id;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS cohort_events_mv
+ENGINE = AggregatingMergeTree()
+ORDER BY (project_id, name, created_at, profile_id)
+AS SELECT
+        project_id,
+        name,
+        toDate(created_at) AS created_at,
+        profile_id,
+        COUNT() AS event_count
+      FROM events
+      WHERE profile_id != device_id
+      GROUP BY project_id, name, created_at, profile_id;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS distinct_event_names_mv
+ENGINE = AggregatingMergeTree()
+ORDER BY (project_id, name, created_at)
+AS SELECT
+        project_id,
+        name,
+        max(created_at) AS created_at,
+        count() AS event_count
+      FROM events
+      GROUP BY project_id, name;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS event_property_values_mv
+ENGINE = AggregatingMergeTree()
+ORDER BY (project_id, name, property_key, property_value)
+AS SELECT
+        project_id,
+        name,
+        key_value.keys as property_key,
+        key_value.values as property_value,
+        created_at
+      FROM (
+        SELECT
+          project_id,
+          name,
+          untuple(arrayJoin(properties)) as key_value,
+          max(created_at) as created_at
+        FROM events
+        GROUP BY project_id, name, key_value
+      )
+      WHERE property_value != ''
+        AND property_key != ''
+        AND property_key NOT IN ('__duration_from', '__properties_from')
+      GROUP BY project_id, name, property_key, property_value, created_at;

packages/db/code-migrations/4-add-sessions.ts

@@ -0,0 +1,159 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { formatClickhouseDate } from '../src/clickhouse/client';
+import {
+  createTable,
+  runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+export async function up() {
+  const isClustered = getIsCluster();
+
+  const sqls: string[] = [
+    ...createTable({
+      name: 'sessions',
+      engine: 'VersionedCollapsingMergeTree(sign, version)',
+      columns: [
+        '`id` String',
+        '`project_id` String CODEC(ZSTD(3))',
+        '`profile_id` String CODEC(ZSTD(3))',
+        '`device_id` String CODEC(ZSTD(3))',
+        '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+        '`ended_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+        '`is_bounce` Bool',
+        '`entry_origin` LowCardinality(String)',
+        '`entry_path` String CODEC(ZSTD(3))',
+        '`exit_origin` LowCardinality(String)',
+        '`exit_path` String CODEC(ZSTD(3))',
+        '`screen_view_count` Int32',
+        '`revenue` Float64',
+        '`event_count` Int32',
+        '`duration` UInt32',
+        '`country` LowCardinality(FixedString(2))',
+        '`region` LowCardinality(String)',
+        '`city` String',
+        '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+        '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+        '`device` LowCardinality(String)',
+        '`brand` LowCardinality(String)',
+        '`model` LowCardinality(String)',
+        '`browser` LowCardinality(String)',
+        '`browser_version` LowCardinality(String)',
+        '`os` LowCardinality(String)',
+        '`os_version` LowCardinality(String)',
+        '`utm_medium` String CODEC(ZSTD(3))',
+        '`utm_source` String CODEC(ZSTD(3))',
+        '`utm_campaign` String CODEC(ZSTD(3))',
+        '`utm_content` String CODEC(ZSTD(3))',
+        '`utm_term` String CODEC(ZSTD(3))',
+        '`referrer` String CODEC(ZSTD(3))',
+        '`referrer_name` String CODEC(ZSTD(3))',
+        '`referrer_type` LowCardinality(String)',
+        '`sign` Int8',
+        '`version` UInt64',
+        '`properties` Map(String, String) CODEC(ZSTD(3))',
+      ],
+      orderBy: ['project_id', 'id', 'toDate(created_at)', 'profile_id'],
+      partitionBy: 'toYYYYMM(created_at)',
+      settings: {
+        index_granularity: 8192,
+      },
+      distributionHash:
+        'cityHash64(project_id, toString(toStartOfHour(created_at)))',
+      replicatedVersion: '1',
+      isClustered,
+    }),
+  ];
+
+  sqls.push(...createOldSessions());
+
+  fs.writeFileSync(
+    path.join(__filename.replace('.ts', '.sql')),
+    sqls
+      .map((sql) =>
+        sql
+          .trim()
+          .replace(/;$/, '')
+          .replace(/\n{2,}/g, '\n')
+          .concat(';'),
+      )
+      .join('\n\n---\n\n'),
+  );
+
+  if (!process.argv.includes('--dry')) {
+    await runClickhouseMigrationCommands(sqls);
+  }
+}
+
+function createOldSessions() {
+  let startDate = new Date('2024-03-01');
+  const endDate = new Date();
+  const sqls: string[] = [];
+  while (startDate <= endDate) {
+    const endDate = startDate;
+    startDate = new Date(startDate.getTime() + 1000 * 60 * 60 * 24);
+    sqls.push(`
+      INSERT INTO openpanel.sessions
+        WITH unique_sessions AS (
+          SELECT session_id, min(created_at) as first_event_at
+          FROM openpanel.events
+          WHERE 
+            created_at BETWEEN '${formatClickhouseDate(endDate)}' AND '${formatClickhouseDate(startDate)}'
+            AND session_id != ''
+          GROUP BY session_id
+          HAVING first_event_at >= '${formatClickhouseDate(endDate)}'
+        )
+        SELECT 
+          any(e.session_id) as id,
+          any(e.project_id) as project_id,
+          if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,
+          any(e.device_id) as device_id,
+          argMin(e.created_at, e.created_at) as created_at,
+          argMax(e.created_at, e.created_at) as ended_at,
+          if(
+            argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = '',
+            if(countIf(e.name = 'screen_view') > 1, true, false),
+            argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = 'true'
+          ) as is_bounce,
+          argMinIf(e.origin, e.created_at, e.name = 'session_start') as entry_origin,
+          argMinIf(e.path, e.created_at, e.name = 'session_start') as entry_path,
+          argMaxIf(e.origin, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_origin,
+          argMaxIf(e.path, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_path,
+          countIf(e.name = 'screen_view') as screen_view_count,
+          0 as revenue,
+          countIf(e.name != 'screen_view' AND e.name != 'session_start' AND e.name != 'session_end') as event_count,
+          sumIf(e.duration, name = 'session_end') AS duration,
+          argMinIf(e.country, e.created_at, e.name = 'session_start') as country,
+          argMinIf(e.region, e.created_at, e.name = 'session_start') as region,
+          argMinIf(e.city, e.created_at, e.name = 'session_start') as city,
+          argMinIf(e.longitude, e.created_at, e.name = 'session_start') as longitude,
+          argMinIf(e.latitude, e.created_at, e.name = 'session_start') as latitude,
+          argMinIf(e.device, e.created_at, e.name = 'session_start') as device,
+          argMinIf(e.brand, e.created_at, e.name = 'session_start') as brand,
+          argMinIf(e.model, e.created_at, e.name = 'session_start') as model,
+          argMinIf(e.browser, e.created_at, e.name = 'session_start') as browser,
+          argMinIf(e.browser_version, e.created_at, e.name = 'session_start') as browser_version,
+          argMinIf(e.os, e.created_at, e.name = 'session_start') as os,
+          argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version,
+          argMinIf(e.properties['__utm_medium'], e.created_at, e.name = 'session_start') as utm_medium,
+          argMinIf(e.properties['__utm_source'], e.created_at, e.name = 'session_start') as utm_source,
+          argMinIf(e.properties['__utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign,
+          argMinIf(e.properties['__utm_content'], e.created_at, e.name = 'session_start') as utm_content,
+          argMinIf(e.properties['__utm_term'], e.created_at, e.name = 'session_start') as utm_term,
+          argMinIf(e.referrer, e.created_at, e.name = 'session_start') as referrer,
+          argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,
+          argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type,
+          1 as sign,
+          1 as version,
+          argMinIf(e.properties, e.created_at, e.name = 'session_start') as properties
+        FROM events e
+        WHERE 
+          e.session_id IN (SELECT session_id FROM unique_sessions)
+          AND e.created_at BETWEEN '${formatClickhouseDate(endDate)}' AND '${formatClickhouseDate(new Date(startDate.getTime() + 1000 * 60 * 60 * 24 * 3))}'
+        GROUP BY e.session_id
+      `);
+  }
+
+  return sqls;
+}

packages/db/code-migrations/helpers.ts

@@ -3,11 +3,30 @@ export function printBoxMessage(title: string, lines: (string | unknown)[]) {
   console.log('│');
   if (title) {
     console.log(`│  ${title}`);
+    if (lines.length) {
       console.log('│');
     }
+  }
   lines.forEach((line) => {
     console.log(`│  ${line}`);
   });
   console.log('│');
   console.log('└──┘');
 }
+
+export function getIsCluster() {
+  const args = process.argv;
+  const noClusterArg = args.includes('--no-cluster');
+  if (noClusterArg) {
+    return false;
+  }
+  return !getIsSelfHosting();
+}
+
+export function getIsSelfHosting() {
+  return !!process.env.SELF_HOSTED;
+}
+
+export function getIsDry() {
+  return process.argv.includes('--dry');
+}

packages/db/code-migrations/migrate.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { ch, db } from '../index';
+import { db } from '../index';
-import { printBoxMessage } from './helpers';
+import { getIsDry, getIsSelfHosting, printBoxMessage } from './helpers';
 
 async function migrate() {
   const args = process.argv.slice(2);
@@ -15,11 +15,38 @@ async function migrate() {
     );
   });
 
+  const finishedMigrations = await db.codeMigration.findMany();
+
+  printBoxMessage('📋 Plan', [
+    '\t✅ Finished:',
+    ...finishedMigrations.map(
+      (migration) => `\t- ${migration.name} (${migration.createdAt})`,
+    ),
+    '',
+    '\t🔄 Will run now:',
+    ...migrations
+      .filter(
+        (migration) =>
+          !finishedMigrations.some(
+            (finishedMigration) => finishedMigration.name === migration,
+          ),
+      )
+      .map((migration) => `\t- ${migration}`),
+  ]);
+
+  printBoxMessage('🌍 Environment', [
+    `POSTGRES:   ${process.env.DATABASE_URL}`,
+    `CLICKHOUSE: ${process.env.CLICKHOUSE_URL}`,
+  ]);
+
+  if (!getIsSelfHosting()) {
+    printBoxMessage('🕒 Migrations starts in 10 seconds', []);
+    await new Promise((resolve) => setTimeout(resolve, 10000));
+  }
+
   if (migration) {
     await runMigration(migrationsDir, migration);
   } else {
-    const finishedMigrations = await db.codeMigration.findMany();
-
     for (const file of migrations) {
       if (finishedMigrations.some((migration) => migration.name === file)) {
         printBoxMessage('✅  Already Migrated  ✅', [`${file}`]);
@@ -39,6 +66,7 @@ async function runMigration(migrationsDir: string, file: string) {
   try {
     const migration = await import(path.join(migrationsDir, file));
     await migration.up();
+    if (!getIsDry()) {
       await db.codeMigration.upsert({
         where: {
           name: file,
@@ -50,6 +78,7 @@ async function runMigration(migrationsDir: string, file: string) {
           name: file,
         },
       });
+    }
   } catch (error) {
     printBoxMessage('❌  Migration Failed  ❌', [
       `Error running migration ${file}:`,

packages/db/index.ts

@@ -11,6 +11,7 @@ export * from './src/services/project.service';
 export * from './src/services/reports.service';
 export * from './src/services/salt.service';
 export * from './src/services/share.service';
+export * from './src/services/session.service';
 export * from './src/services/user.service';
 export * from './src/services/reference.service';
 export * from './src/services/id.service';
@@ -18,3 +19,5 @@ export * from './src/services/retention.service';
 export * from './src/services/notification.service';
 export * from './src/buffers';
 export * from './src/types';
+export * from './src/clickhouse/query-builder';
+export * from './src/services/overview.service';

packages/db/package.json

@@ -28,7 +28,8 @@
     "ramda": "^0.29.1",
     "sqlstring": "^2.3.3",
     "superjson": "^1.13.3",
-    "uuid": "^9.0.1"
+    "uuid": "^9.0.1",
+    "zod": "^3.22.4"
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",

packages/db/src/buffers/index.ts

@@ -1,7 +1,9 @@
 import { BotBuffer as BotBufferRedis } from './bot-buffer-redis';
 import { EventBuffer as EventBufferRedis } from './event-buffer-redis';
 import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer-redis';
+import { SessionBuffer } from './session-buffer';
 
 export const eventBuffer = new EventBufferRedis();
 export const profileBuffer = new ProfileBufferRedis();
 export const botBuffer = new BotBufferRedis();
+export const sessionBuffer = new SessionBuffer();

packages/db/src/buffers/session-buffer.ts

@@ -0,0 +1,211 @@
+import { type Redis, getRedisCache, runEvery } from '@openpanel/redis';
+
+import { toDots } from '@openpanel/common';
+import { getSafeJson } from '@openpanel/json';
+import { assocPath, clone } from 'ramda';
+import { TABLE_NAMES, ch } from '../clickhouse/client';
+import type { IClickhouseEvent } from '../services/event.service';
+import type { IClickhouseSession } from '../services/session.service';
+import { BaseBuffer } from './base-buffer';
+
+export class SessionBuffer extends BaseBuffer {
+  private batchSize = process.env.BOT_BUFFER_BATCH_SIZE
+    ? Number.parseInt(process.env.BOT_BUFFER_BATCH_SIZE, 10)
+    : 2;
+
+  private readonly redisKey = 'session-buffer';
+  private redis: Redis;
+  constructor() {
+    super({
+      name: 'session',
+      onFlush: async () => {
+        await this.processBuffer();
+      },
+    });
+    this.redis = getRedisCache();
+  }
+
+  async getExistingSession(sessionId: string) {
+    const hit = await this.redis.get(`session:${sessionId}`);
+
+    if (hit) {
+      return getSafeJson<IClickhouseSession>(hit);
+    }
+
+    return null;
+  }
+
+  async getSession(
+    event: IClickhouseEvent,
+  ): Promise<[IClickhouseSession] | [IClickhouseSession, IClickhouseSession]> {
+    const existingSession = await this.getExistingSession(event.session_id);
+
+    if (existingSession) {
+      const oldSession = assocPath(['sign'], -1, clone(existingSession));
+      const newSession = assocPath(['sign'], 1, clone(existingSession));
+
+      newSession.ended_at = event.created_at;
+      newSession.version = existingSession.version + 1;
+      if (!newSession.entry_path) {
+        newSession.entry_path = event.path;
+        newSession.entry_origin = event.origin;
+      }
+      newSession.exit_path = event.path;
+      newSession.exit_origin = event.origin;
+      newSession.duration =
+        new Date(newSession.ended_at).getTime() -
+        new Date(newSession.created_at).getTime();
+      newSession.properties = toDots({
+        ...(event.properties || {}),
+        ...(newSession.properties || {}),
+      });
+      // newSession.revenue += event.properties?.__revenue ?? 0;
+
+      if (event.name === 'screen_view') {
+        newSession.screen_views.push(event.path);
+        newSession.screen_view_count += 1;
+      } else {
+        newSession.event_count += 1;
+      }
+
+      if (newSession.screen_view_count > 1) {
+        newSession.is_bounce = false;
+      }
+
+      // If the profile_id is set and it's different from the device_id, we need to update the profile_id
+      if (event.profile_id && event.profile_id !== event.device_id) {
+        newSession.profile_id = event.profile_id;
+      }
+
+      return [newSession, oldSession];
+    }
+
+    return [
+      {
+        id: event.session_id,
+        is_bounce: true,
+        profile_id: event.profile_id,
+        project_id: event.project_id,
+        device_id: event.device_id,
+        created_at: event.created_at,
+        ended_at: event.created_at,
+        event_count: event.name === 'screen_view' ? 0 : 1,
+        screen_view_count: event.name === 'screen_view' ? 1 : 0,
+        screen_views: event.name === 'screen_view' ? [event.path] : [],
+        entry_path: event.path,
+        entry_origin: event.origin,
+        exit_path: event.path,
+        exit_origin: event.origin,
+        revenue: 0,
+        referrer: event.referrer,
+        referrer_name: event.referrer_name,
+        referrer_type: event.referrer_type,
+        os: event.os,
+        os_version: event.os_version,
+        browser: event.browser,
+        browser_version: event.browser_version,
+        device: event.device,
+        brand: event.brand,
+        model: event.model,
+        country: event.country,
+        region: event.region,
+        city: event.city,
+        longitude: event.longitude ?? null,
+        latitude: event.latitude ?? null,
+        duration: event.duration,
+        utm_medium: event.properties?.['__query.utm_medium']
+          ? String(event.properties?.['__query.utm_medium'])
+          : '',
+        utm_source: event.properties?.['__query.utm_source']
+          ? String(event.properties?.['__query.utm_source'])
+          : '',
+        utm_campaign: event.properties?.['__query.utm_campaign']
+          ? String(event.properties?.['__query.utm_campaign'])
+          : '',
+        utm_content: event.properties?.['__query.utm_content']
+          ? String(event.properties?.['__query.utm_content'])
+          : '',
+        utm_term: event.properties?.['__query.utm_term']
+          ? String(event.properties?.['__query.utm_term'])
+          : '',
+        sign: 1,
+        version: 1,
+        properties: toDots(event.properties || {}),
+      },
+    ];
+  }
+
+  async add(event: IClickhouseEvent) {
+    if (!event.session_id) {
+      return;
+    }
+
+    if (['session_start', 'session_end'].includes(event.name)) {
+      return;
+    }
+
+    try {
+      // Plural since we will delete the old session with sign column
+      const sessions = await this.getSession(event);
+      const [newSession] = sessions;
+
+      console.log(`Adding sessions ${sessions.length}`);
+
+      const multi = this.redis.multi();
+      multi.set(
+        `session:${newSession.id}`,
+        JSON.stringify(newSession),
+        'EX',
+        60 * 60,
+      );
+      for (const session of sessions) {
+        multi.rpush(this.redisKey, JSON.stringify(session));
+      }
+      await multi.exec();
+
+      // Check buffer length
+      const bufferLength = await this.redis.llen(this.redisKey);
+
+      if (bufferLength >= this.batchSize) {
+        await this.tryFlush();
+      }
+    } catch (error) {
+      this.logger.error('Failed to add bot event', { error });
+    }
+  }
+
+  async processBuffer() {
+    try {
+      // Get events from the start without removing them
+      const events = await this.redis.lrange(
+        this.redisKey,
+        0,
+        this.batchSize - 1,
+      );
+
+      if (events.length === 0) return;
+
+      const sessions = events.map((e) => getSafeJson<IClickhouseSession>(e));
+
+      // Insert to ClickHouse
+      await ch.insert({
+        table: TABLE_NAMES.sessions,
+        values: sessions,
+        format: 'JSONEachRow',
+      });
+
+      // Only remove events after successful insert
+      await this.redis.ltrim(this.redisKey, events.length, -1);
+
+      this.logger.info('Processed sessions', {
+        count: events.length,
+      });
+    } catch (error) {
+      this.logger.error('Failed to process buffer', { error });
+    }
+  }
+
+  async getBufferSize() {
+    return getRedisCache().llen(this.redisKey);
+  }
+}

packages/db/src/clickhouse/client.ts

@@ -11,6 +11,7 @@ export { createClient };
 const logger = createLogger({ name: 'clickhouse' });
 
 import type { Logger } from '@clickhouse/client';
+import { getTimezoneFromDateString } from '@openpanel/common';
 
 // All three LogParams types are exported by the client
 interface LogParams {
@@ -55,6 +56,7 @@ export const TABLE_NAMES = {
   event_names_mv: 'distinct_event_names_mv',
   event_property_values_mv: 'event_property_values_mv',
   cohort_events_mv: 'cohort_events_mv',
+  sessions: 'sessions',
 };
 
 export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {

packages/db/src/clickhouse/migration.ts

@@ -39,10 +39,10 @@ export const chMigrationClient = createClient({
   request_timeout: 3600000, // 1 hour in milliseconds
   keep_alive: {
     enabled: true,
-    idle_socket_ttl: 8000,
   },
   compression: {
     request: true,
+    response: true,
   },
   clickhouse_settings: {
     wait_end_of_query: 1,

packages/db/src/clickhouse/query-builder.ts

@@ -0,0 +1,730 @@
+import type { ClickHouseClient, ResponseJSON } from '@clickhouse/client';
+import type { IInterval } from '@openpanel/validation';
+import { escape } from 'sqlstring';
+
+type SqlValue = string | number | boolean | Date | null | Expression;
+type SqlParam = SqlValue | SqlValue[];
+type Operator =
+  | '='
+  | '>'
+  | '<'
+  | '>='
+  | '<='
+  | '!='
+  | 'IN'
+  | 'NOT IN'
+  | 'LIKE'
+  | 'NOT LIKE'
+  | 'IS NULL'
+  | 'IS NOT NULL'
+  | 'BETWEEN';
+
+type CTE = {
+  name: string;
+  query: Query | string;
+};
+
+type JoinType = 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' | 'CROSS';
+
+type WhereCondition = {
+  condition: string;
+  operator: 'AND' | 'OR';
+  isGroup?: boolean;
+};
+
+type ConditionalCallback = (query: Query) => void;
+
+class Expression {
+  constructor(private expression: string) {}
+
+  toString() {
+    return this.expression;
+  }
+}
+
+export class Query<T = any> {
+  private _select: string[] = [];
+  private _except: string[] = [];
+  private _from?: string | Expression;
+  private _where: WhereCondition[] = [];
+  private _groupBy: string[] = [];
+  private _rollup = false;
+  private _having: { condition: string; operator: 'AND' | 'OR' }[] = [];
+  private _orderBy: {
+    column: string;
+    direction: 'ASC' | 'DESC';
+  }[] = [];
+  private _limit?: number;
+  private _offset?: number;
+  private _final = false;
+  private _settings: Record<string, string> = {};
+  private _ctes: CTE[] = [];
+  private _joins: {
+    type: JoinType;
+    table: string | Expression;
+    condition: string;
+    alias?: string;
+  }[] = [];
+  private _skipNext = false;
+  private _fill?: {
+    from: string | Date;
+    to: string | Date;
+    step: string;
+  };
+  private _transform?: Record<string, (item: T) => any>;
+  private _union?: Query;
+  constructor(private client: ClickHouseClient) {}
+
+  // Select methods
+  select<U>(
+    columns: (string | null | undefined | false)[],
+    type: 'merge' | 'replace' = 'replace',
+  ): Query<U> {
+    if (this._skipNext) return this as unknown as Query<U>;
+    if (type === 'merge') {
+      this._select = [
+        ...this._select,
+        ...columns.filter((col): col is string => Boolean(col)),
+      ];
+    } else {
+      this._select = columns.filter((col): col is string => Boolean(col));
+    }
+    return this as unknown as Query<U>;
+  }
+
+  except(columns: string[]): this {
+    this._except = [...this._except, ...columns];
+    return this;
+  }
+
+  rollup(): this {
+    this._rollup = true;
+    return this;
+  }
+
+  // From methods
+  from(table: string | Expression, final = false): this {
+    this._from = table;
+    this._final = final;
+    return this;
+  }
+
+  union(query: Query): this {
+    this._union = query;
+    return this;
+  }
+
+  // Where methods
+  private escapeValue(value: SqlParam): string {
+    if (value === null) return 'NULL';
+    if (value instanceof Expression) return `(${value.toString()})`;
+    if (Array.isArray(value)) {
+      return `(${value.map((v) => this.escapeValue(v)).join(', ')})`;
+    }
+    if (value instanceof Date) {
+      return escape(clix.datetime(value));
+    }
+    return escape(value);
+  }
+
+  where(column: string, operator: Operator, value?: SqlParam): this {
+    if (this._skipNext) return this;
+    const condition = this.buildCondition(column, operator, value);
+    this._where.push({ condition, operator: 'AND' });
+    return this;
+  }
+
+  public buildCondition(
+    column: string,
+    operator: Operator,
+    value?: SqlParam,
+  ): string {
+    switch (operator) {
+      case 'IS NULL':
+        return `${column} IS NULL`;
+      case 'IS NOT NULL':
+        return `${column} IS NOT NULL`;
+      case 'BETWEEN':
+        if (Array.isArray(value) && value.length === 2) {
+          return `${column} BETWEEN ${this.escapeValue(value[0]!)} AND ${this.escapeValue(value[1]!)}`;
+        }
+        throw new Error('BETWEEN operator requires an array of two values');
+      case 'IN':
+      case 'NOT IN':
+        if (!Array.isArray(value) && !(value instanceof Expression)) {
+          throw new Error(`${operator} operator requires an array value`);
+        }
+        return `${column} ${operator} (${this.escapeValue(value)})`;
+      default:
+        return `${column} ${operator} ${this.escapeValue(value!)}`;
+    }
+  }
+
+  andWhere(column: string, operator: Operator, value?: SqlParam): this {
+    const condition = this.buildCondition(column, operator, value);
+    this._where.push({ condition, operator: 'AND' });
+    return this;
+  }
+
+  rawWhere(condition: string): this {
+    if (condition) {
+      this._where.push({ condition, operator: 'AND' });
+    }
+    return this;
+  }
+
+  orWhere(column: string, operator: Operator, value?: SqlParam): this {
+    const condition = this.buildCondition(column, operator, value);
+    this._where.push({ condition, operator: 'OR' });
+    return this;
+  }
+
+  // Group by methods
+  groupBy(columns: (string | null | undefined | false)[]): this {
+    this._groupBy = columns.filter((col): col is string => Boolean(col));
+    return this;
+  }
+
+  // Having methods
+  having(column: string, operator: Operator, value: SqlParam): this {
+    const condition = this.buildCondition(column, operator, value);
+    this._having.push({ condition, operator: 'AND' });
+    return this;
+  }
+
+  andHaving(column: string, operator: Operator, value: SqlParam): this {
+    const condition = this.buildCondition(column, operator, value);
+    this._having.push({ condition, operator: 'AND' });
+    return this;
+  }
+
+  orHaving(column: string, operator: Operator, value: SqlParam): this {
+    const condition = this.buildCondition(column, operator, value);
+    this._having.push({ condition, operator: 'OR' });
+    return this;
+  }
+
+  // Order by methods
+  orderBy(column: string, direction: 'ASC' | 'DESC' = 'ASC'): this {
+    if (this._skipNext) return this;
+    this._orderBy.push({ column, direction });
+    return this;
+  }
+
+  // Limit and offset
+  limit(limit?: number): this {
+    if (limit !== undefined) {
+      this._limit = limit;
+    }
+    return this;
+  }
+
+  offset(offset?: number): this {
+    if (offset !== undefined) {
+      this._offset = offset;
+    }
+    return this;
+  }
+
+  // Settings
+  settings(settings: Record<string, string>): this {
+    Object.assign(this._settings, settings);
+    return this;
+  }
+
+  with(name: string, query: Query | string): this {
+    this._ctes.push({ name, query });
+    return this;
+  }
+
+  // Fill
+  fill(from: string | Date, to: string | Date, step: string): this {
+    this._fill = {
+      from: this.escapeDate(from),
+      to: this.escapeDate(to),
+      step: step,
+    };
+    return this;
+  }
+
+  private escapeDate(value: string | Date): string {
+    if (value instanceof Date) {
+      return clix.datetime(value);
+    }
+
+    return value.replaceAll(/\d{4}-\d{2}-\d{2}([\s\:\d\.]+)?/g, (match) => {
+      return escape(match);
+    });
+  }
+
+  // Add join methods
+  join(table: string | Expression, condition: string, alias?: string): this {
+    return this.joinWithType('INNER', table, condition, alias);
+  }
+
+  innerJoin(
+    table: string | Expression,
+    condition: string,
+    alias?: string,
+  ): this {
+    return this.joinWithType('INNER', table, condition, alias);
+  }
+
+  leftJoin(
+    table: string | Expression,
+    condition: string,
+    alias?: string,
+  ): this {
+    return this.joinWithType('LEFT', table, condition, alias);
+  }
+
+  rightJoin(
+    table: string | Expression,
+    condition: string,
+    alias?: string,
+  ): this {
+    return this.joinWithType('RIGHT', table, condition, alias);
+  }
+
+  fullJoin(
+    table: string | Expression,
+    condition: string,
+    alias?: string,
+  ): this {
+    return this.joinWithType('FULL', table, condition, alias);
+  }
+
+  crossJoin(table: string | Expression, alias?: string): this {
+    return this.joinWithType('CROSS', table, '', alias);
+  }
+
+  private joinWithType(
+    type: JoinType,
+    table: string | Expression,
+    condition: string,
+    alias?: string,
+  ): this {
+    if (this._skipNext) return this;
+    this._joins.push({
+      type,
+      table,
+      condition: this.escapeDate(condition),
+      alias,
+    });
+    return this;
+  }
+
+  // Add methods for grouping conditions
+  whereGroup(): WhereGroupBuilder {
+    return new WhereGroupBuilder(this, 'AND');
+  }
+
+  orWhereGroup(): WhereGroupBuilder {
+    return new WhereGroupBuilder(this, 'OR');
+  }
+
+  // Update buildQuery method's WHERE section
+  private buildWhereConditions(conditions: WhereCondition[]): string {
+    return conditions
+      .map((w, i) => {
+        const condition = w.isGroup ? `(${w.condition})` : w.condition;
+        return i === 0 ? condition : `${w.operator} ${condition}`;
+      })
+      .join(' ');
+  }
+
+  private buildQuery(): string {
+    const parts: string[] = [];
+
+    // Add WITH clause if CTEs exist
+    if (this._ctes.length > 0) {
+      const cteStatements = this._ctes.map((cte) => {
+        const queryStr =
+          typeof cte.query === 'string' ? cte.query : cte.query.toSQL();
+        return `${cte.name} AS (${queryStr})`;
+      });
+      parts.push(`WITH ${cteStatements.join(', ')}`);
+    }
+
+    // SELECT
+    if (this._select.length > 0) {
+      parts.push('SELECT', this._select.map(this.escapeDate).join(', '));
+    } else {
+      parts.push('SELECT *');
+    }
+
+    if (this._except.length > 0) {
+      parts.push('EXCEPT', `(${this._except.map(this.escapeDate).join(', ')})`);
+    }
+
+    // FROM
+    if (this._from) {
+      if (this._from instanceof Expression) {
+        parts.push(`FROM (${this._from.toString()})`);
+      } else {
+        parts.push(`FROM ${this._from}${this._final ? ' FINAL' : ''}`);
+      }
+
+      // Add joins
+      this._joins.forEach((join) => {
+        const aliasClause = join.alias ? ` ${join.alias} ` : ' ';
+        const conditionStr = join.condition ? `ON ${join.condition}` : '';
+        parts.push(
+          `${join.type} JOIN ${join.table instanceof Expression ? `(${join.table.toString()})` : join.table}${aliasClause}${conditionStr}`,
+        );
+      });
+    }
+
+    // WHERE
+    if (this._where.length > 0) {
+      parts.push('WHERE', this.buildWhereConditions(this._where));
+    }
+
+    // GROUP BY
+    if (this._groupBy.length > 0) {
+      parts.push('GROUP BY', this._groupBy.join(', '));
+    }
+
+    if (this._rollup) {
+      parts.push('WITH ROLLUP');
+    }
+
+    // HAVING
+    if (this._having.length > 0) {
+      const conditions = this._having.map((h, i) => {
+        return i === 0 ? h.condition : `${h.operator} ${h.condition}`;
+      });
+      parts.push('HAVING', conditions.join(' '));
+    }
+
+    // ORDER BY
+    if (this._orderBy.length > 0) {
+      const orderBy = this._orderBy.map((o) => {
+        const col = o.column;
+        return `${col} ${o.direction}`;
+      });
+      parts.push('ORDER BY', orderBy.join(', '));
+    }
+
+    // Add FILL clause after ORDER BY
+    if (this._fill) {
+      const fromDate =
+        this._fill.from instanceof Date
+          ? clix.datetime(this._fill.from)
+          : this._fill.from;
+      const toDate =
+        this._fill.to instanceof Date
+          ? clix.datetime(this._fill.to)
+          : this._fill.to;
+
+      parts.push('WITH FILL');
+      parts.push(`FROM ${fromDate}`);
+      parts.push(`TO ${toDate}`);
+      parts.push(`STEP ${this._fill.step}`);
+    }
+
+    // LIMIT & OFFSET
+    if (this._limit !== undefined) {
+      parts.push(`LIMIT ${this._limit}`);
+      if (this._offset !== undefined) {
+        parts.push(`OFFSET ${this._offset}`);
+      }
+    }
+
+    // SETTINGS
+    if (Object.keys(this._settings).length > 0) {
+      const settings = Object.entries(this._settings)
+        .map(([key, value]) => `${key} = ${value}`)
+        .join(', ');
+      parts.push(`SETTINGS ${settings}`);
+    }
+
+    if (this._union) {
+      parts.push('UNION ALL', this._union.buildQuery());
+    }
+
+    return parts.join(' ');
+  }
+
+  transformJson<E extends ResponseJSON<any>>(json: E): E {
+    const keys = Object.keys(json.data[0] || {});
+    const response = {
+      ...json,
+      data: json.data.map((item) => {
+        return keys.reduce((acc, key) => {
+          const meta = json.meta?.find((m) => m.name === key);
+          const transformer = this._transform?.[key];
+
+          if (transformer) {
+            return {
+              ...acc,
+              [key]: transformer(item),
+            };
+          }
+
+          return {
+            ...acc,
+            [key]:
+              item[key] && meta?.type.includes('Int')
+                ? Number.parseFloat(item[key] as string)
+                : item[key],
+          };
+        }, {} as T);
+      }),
+    };
+    return response;
+  }
+
+  transform(transformations: Record<string, (item: T) => any>): this {
+    this._transform = transformations;
+    return this;
+  }
+
+  // Execution methods
+  async execute(): Promise<T[]> {
+    const query = this.buildQuery();
+    console.log('TEST QUERY ----->');
+    console.log(query);
+    console.log('<----------');
+    const perf = performance.now();
+    try {
+      const result = await this.client.query({
+        query,
+      });
+      const json = await result.json<T>();
+      const perf2 = performance.now();
+      console.log(`PERF: ${perf2 - perf}ms`);
+      return this.transformJson(json).data;
+    } catch (error) {
+      console.log('ERROR ----->');
+      console.log(error);
+      console.log('<----------');
+      console.log(query);
+      console.log('<----------');
+      throw error;
+    }
+  }
+
+  // Debug methods
+  toSQL(): string {
+    return this.buildQuery();
+  }
+
+  // Add method to add where conditions (for internal use)
+  _addWhereCondition(condition: WhereCondition): this {
+    this._where.push(condition);
+    return this;
+  }
+
+  if(condition: any): this {
+    this._skipNext = !condition;
+    return this;
+  }
+
+  endIf(): this {
+    this._skipNext = false;
+    return this;
+  }
+
+  // Add method for callback-style conditionals
+  when(condition: boolean, callback?: ConditionalCallback): this {
+    if (condition && callback) {
+      callback(this);
+    }
+    return this;
+  }
+
+  clone(): Query<T> {
+    return new Query(this.client).merge(this);
+  }
+
+  // Add merge method
+  merge(query: Query): this {
+    if (this._skipNext) return this;
+
+    this._from = query._from;
+
+    this._select = [...this._select, ...query._select];
+
+    this._except = [...this._except, ...query._except];
+
+    // Merge WHERE conditions
+    this._where = [...this._where, ...query._where];
+
+    // Merge CTEs
+    this._ctes = [...this._ctes, ...query._ctes];
+
+    // Merge JOINS
+    this._joins = [...this._joins, ...query._joins];
+
+    // Merge settings
+    this._settings = { ...this._settings, ...query._settings };
+
+    // Take the most restrictive LIMIT
+    if (query._limit !== undefined) {
+      this._limit =
+        this._limit === undefined
+          ? query._limit
+          : Math.min(this._limit, query._limit);
+    }
+
+    // Merge ORDER BY
+    this._orderBy = [...this._orderBy, ...query._orderBy];
+
+    // Merge GROUP BY
+    this._groupBy = [...this._groupBy, ...query._groupBy];
+
+    // Merge HAVING conditions
+    this._having = [...this._having, ...query._having];
+
+    return this;
+  }
+}
+
+// Add this new class for building where groups
+export class WhereGroupBuilder {
+  private conditions: WhereCondition[] = [];
+
+  constructor(
+    private query: Query,
+    private groupOperator: 'AND' | 'OR',
+  ) {}
+
+  where(column: string, operator: Operator, value?: SqlParam): this {
+    const condition = this.query.buildCondition(column, operator, value);
+    this.conditions.push({ condition, operator: 'AND' });
+    return this;
+  }
+
+  andWhere(column: string, operator: Operator, value?: SqlParam): this {
+    const condition = this.query.buildCondition(column, operator, value);
+    this.conditions.push({ condition, operator: 'AND' });
+    return this;
+  }
+
+  rawWhere(condition: string): this {
+    this.conditions.push({ condition, operator: 'AND' });
+    return this;
+  }
+
+  orWhere(column: string, operator: Operator, value?: SqlParam): this {
+    const condition = this.query.buildCondition(column, operator, value);
+    this.conditions.push({ condition, operator: 'OR' });
+    return this;
+  }
+
+  end(): Query {
+    const groupCondition = this.conditions
+      .map((c, i) => (i === 0 ? c.condition : `${c.operator} ${c.condition}`))
+      .join(' ');
+
+    this.query._addWhereCondition({
+      condition: groupCondition,
+      operator: this.groupOperator,
+      isGroup: true,
+    });
+
+    return this.query;
+  }
+}
+
+// Helper function to create a new query
+export function createQuery(client: ClickHouseClient): Query {
+  return new Query(client);
+}
+
+export function clix(client: ClickHouseClient): Query {
+  return new Query(client);
+}
+
+clix.exp = (expr: string | Query<any>) =>
+  new Expression(expr instanceof Query ? expr.toSQL() : expr);
+clix.date = (date: string | Date, wrapper?: string) => {
+  const dateStr = new Date(date).toISOString().slice(0, 10);
+  return wrapper ? `${wrapper}(${dateStr})` : dateStr;
+};
+clix.datetime = (date: string | Date, wrapper?: string) => {
+  const datetime = new Date(date).toISOString().slice(0, 19).replace('T', ' ');
+  return wrapper ? `${wrapper}(${datetime})` : datetime;
+};
+clix.dynamicDatetime = (date: string | Date, interval: IInterval) => {
+  if (interval === 'month' || interval === 'week') {
+    return clix.date(date);
+  }
+  return clix.datetime(date);
+};
+
+clix.toStartOf = (node: string, interval: IInterval) => {
+  switch (interval) {
+    case 'minute': {
+      return `toStartOfMinute(${node})`;
+    }
+    case 'hour': {
+      return `toStartOfHour(${node})`;
+    }
+    case 'day': {
+      return `toStartOfDay(${node})`;
+    }
+    case 'week': {
+      return `toStartOfWeek(${node})`;
+    }
+    case 'month': {
+      return `toStartOfMonth(${node})`;
+    }
+  }
+};
+clix.toStartOfInterval = (
+  node: string,
+  interval: IInterval,
+  origin: string | Date,
+) => {
+  switch (interval) {
+    case 'minute': {
+      return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 minute, toDateTime(${clix.datetime(origin)}))`;
+    }
+    case 'hour': {
+      return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 hour, toDateTime(${clix.datetime(origin)}))`;
+    }
+    case 'day': {
+      return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 day, toDateTime(${clix.datetime(origin)}))`;
+    }
+    case 'week': {
+      return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 week, toDateTime(${clix.datetime(origin)}))`;
+    }
+    case 'month': {
+      return `toStartOfInterval(toDateTime(${node}), INTERVAL 1 month, toDateTime(${clix.datetime(origin)}))`;
+    }
+  }
+};
+clix.toInterval = (node: string, interval: IInterval) => {
+  switch (interval) {
+    case 'minute': {
+      return `toIntervalMinute(${node})`;
+    }
+    case 'hour': {
+      return `toIntervalHour(${node})`;
+    }
+    case 'day': {
+      return `toIntervalDay(${node})`;
+    }
+    case 'week': {
+      return `toIntervalWeek(${node})`;
+    }
+    case 'month': {
+      return `toIntervalMonth(${node})`;
+    }
+  }
+};
+clix.toDate = (node: string, interval: IInterval) => {
+  switch (interval) {
+    case 'week':
+    case 'month': {
+      return `toDate(${node})`;
+    }
+    default: {
+      return `toDateTime(${node})`;
+    }
+  }
+};
+// Export types
+export type { SqlValue, SqlParam, Operator };

packages/db/src/services/chart.service.ts

@@ -17,7 +17,15 @@ import {
 import { createSqlBuilder } from '../sql-builder';
 
 export function transformPropertyKey(property: string) {
-  if (property.startsWith('properties.')) {
+  const propertyPatterns = ['properties', 'profile.properties'];
+  const match = propertyPatterns.find((pattern) =>
+    property.startsWith(`${pattern}.`),
+  );
+
+  if (!match) {
+    return property;
+  }
+
   if (property.includes('*')) {
     return property
       .replace(/^properties\./, '')
@@ -25,23 +33,25 @@ export function transformPropertyKey(property: string) {
       .replace(/\[\*\]$/, '.%')
       .replace(/\[\*\].?/, '.%.');
   }
-    return `properties['${property.replace(/^properties\./, '')}']`;
-  }
 
-  return property;
+  return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
 }
 
 export function getSelectPropertyKey(property: string) {
-  if (property.startsWith('properties.')) {
+  const propertyPatterns = ['properties', 'profile.properties'];
+
+  const match = propertyPatterns.find((pattern) =>
+    property.startsWith(`${pattern}.`),
+  );
+  if (!match) return property;
+
   if (property.includes('*')) {
-      return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
+    return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${escape(
       transformPropertyKey(property),
     )})))`;
   }
-    return `properties['${property.replace(/^properties\./, '')}']`;
-  }
 
-  return property;
+  return `${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;
 }
 
 export function getChartSql({
@@ -54,8 +64,16 @@ export function getChartSql({
   chartType,
   limit,
 }: IGetChartDataInput) {
-  const { sb, join, getWhere, getFrom, getSelect, getOrderBy, getGroupBy } =
+  const {
-    createSqlBuilder();
+    sb,
+    join,
+    getWhere,
+    getFrom,
+    getJoins,
+    getSelect,
+    getOrderBy,
+    getGroupBy,
+  } = createSqlBuilder();
 
   sb.where = getEventFiltersWhereClause(event.filters);
   sb.where.projectId = `project_id = ${escape(projectId)}`;
@@ -67,6 +85,14 @@ export function getChartSql({
     sb.select.label_0 = `'*' as label_0`;
   }
 
+  // const anyFilterOnProfile = event.filters.some((filter) =>
+  //   filter.name.startsWith('profile.properties.'),
+  // );
+
+  // if (anyFilterOnProfile) {
+  //   sb.joins.profiles = 'JOIN profiles profile ON e.profile_id = profile.id';
+  // }
+
   sb.select.count = 'count(*) as count';
   switch (interval) {
     case 'minute': {
@@ -149,10 +175,18 @@ export function getChartSql({
         ORDER BY profile_id, created_at DESC
       ) as subQuery`;
 
-    return `${getSelect()} ${getFrom()} ${getGroupBy()} ${getOrderBy()}`;
+    console.log(
+      `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
+    );
+
+    return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
   }
 
-  return `${getSelect()} ${getFrom()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
+  console.log(
+    `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`,
+  );
+
+  return `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`;
 }
 
 export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
@@ -161,7 +195,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
     const id = `f${index}`;
     const { name, value, operator } = filter;
 
-    if (value.length === 0) return;
+    if (
+      value.length === 0 &&
+      operator !== 'isNull' &&
+      operator !== 'isNotNull'
+    ) {
+      return;
+    }
 
     if (name === 'has_profile') {
       if (value.includes('true')) {
@@ -172,7 +212,10 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
       return;
     }
 
-    if (name.startsWith('properties.')) {
+    if (
+      name.startsWith('properties.') ||
+      name.startsWith('profile.properties.')
+    ) {
       const propertyKey = getSelectPropertyKey(name);
       const isWildcard = propertyKey.includes('%');
       const whereFrom = getSelectPropertyKey(name);
@@ -284,6 +327,23 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
           }
           break;
         }
+        case 'isNull': {
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> x = '' OR x IS NULL, ${whereFrom})`;
+          } else {
+            where[id] = `(${whereFrom} = '' OR ${whereFrom} IS NULL)`;
+          }
+          break;
+        }
+        case 'isNotNull': {
+          if (isWildcard) {
+            where[id] =
+              `arrayExists(x -> x != '' AND x IS NOT NULL, ${whereFrom})`;
+          } else {
+            where[id] = `(${whereFrom} != '' AND ${whereFrom} IS NOT NULL)`;
+          }
+          break;
+        }
       }
     } else {
       switch (operator) {
@@ -297,6 +357,14 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
           }
           break;
         }
+        case 'isNull': {
+          where[id] = `(${name} = '' OR ${name} IS NULL)`;
+          break;
+        }
+        case 'isNotNull': {
+          where[id] = `(${name} != '' AND ${name} IS NOT NULL)`;
+          break;
+        }
         case 'isNot': {
           if (value.length === 1) {
             where[id] = `${name} != ${escape(String(value[0]).trim())}`;

packages/db/src/services/event.service.ts

@@ -1,24 +1,30 @@
-import { mergeDeepRight, uniq } from 'ramda';
+import { path, assocPath, last, mergeDeepRight, pick, uniq } from 'ramda';
 import { escape } from 'sqlstring';
 import { v4 as uuid } from 'uuid';
 
 import { toDots } from '@openpanel/common';
-import { cacheable } from '@openpanel/redis';
+import { cacheable, getCache } from '@openpanel/redis';
 import type { IChartEventFilter } from '@openpanel/validation';
 
-import { botBuffer, eventBuffer } from '../buffers';
+import { botBuffer, eventBuffer, sessionBuffer } from '../buffers';
 import {
   TABLE_NAMES,
+  ch,
   chQuery,
   convertClickhouseDateToJs,
   formatClickhouseDate,
 } from '../clickhouse/client';
+import { type Query, clix } from '../clickhouse/query-builder';
 import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 import { createSqlBuilder } from '../sql-builder';
 import { getEventFiltersWhereClause } from './chart.service';
-import type { IServiceProfile } from './profile.service';
+import type { IClickhouseProfile, IServiceProfile } from './profile.service';
-import { getProfiles, upsertProfile } from './profile.service';
+import {
+  getProfiles,
+  transformProfile,
+  upsertProfile,
+} from './profile.service';
 
 export type IImportedEvent = Omit<
   IClickhouseEvent,
@@ -120,11 +126,11 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
     referrer: event.referrer,
     referrerName: event.referrer_name,
     referrerType: event.referrer_type,
-    profile: event.profile,
     meta: event.meta,
     importedAt: event.imported_at ? new Date(event.imported_at) : undefined,
     sdkName: event.sdk_name,
     sdkVersion: event.sdk_version,
+    profile: event.profile,
   };
 }
 
@@ -246,24 +252,34 @@ export async function getEvents(
     const ids = events.map((e) => e.profile_id);
     const profiles = await getProfiles(ids, projectId);
 
+    const map = new Map<string, IServiceProfile>();
+    for (const profile of profiles) {
+      map.set(profile.id, profile);
+    }
+
     for (const event of events) {
-      event.profile = profiles.find((p) => p.id === event.profile_id);
+      event.profile = map.get(event.profile_id);
     }
   }
 
   if (options.meta && projectId) {
-    const names = uniq(events.map((e) => e.name));
+    const metas = await getCache(
-    const metas = await db.eventMeta.findMany({
+      `event-metas-${projectId}`,
+      60 * 5,
+      async () => {
+        return db.eventMeta.findMany({
           where: {
-        name: {
-          in: names,
-        },
             projectId,
           },
-      select: options.meta === true ? undefined : options.meta,
         });
+      },
+    );
+    const map = new Map<string, EventMeta>();
+    for (const meta of metas) {
+      map.set(meta.name, meta);
+    }
     for (const event of events) {
-      event.meta = metas.find((m) => m.name === event.name);
+      event.meta = map.get(event.name);
     }
   }
   return events.map(transformEvent);
@@ -339,7 +355,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
     sdk_version: payload.sdkVersion ?? '',
   };
 
-  await eventBuffer.add(event);
+  await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);
 
   return {
     document: event,
@@ -350,7 +366,7 @@ export interface GetEventListOptions {
   projectId: string;
   profileId?: string;
   take: number;
-  cursor?: number;
+  cursor?: number | Date;
   events?: string[] | null;
   filters?: IChartEventFilter[];
   startDate?: Date;
@@ -371,8 +387,13 @@ export async function getEventList({
 }: GetEventListOptions) {
   const { sb, getSql, join } = createSqlBuilder();
 
-  sb.limit = take;
+  if (typeof cursor === 'number') {
     sb.offset = Math.max(0, (cursor ?? 0) * take);
+  } else if (cursor instanceof Date) {
+    sb.where.cursor = `created_at <= '${formatClickhouseDate(cursor)}'`;
+  }
+
+  sb.limit = take;
   sb.where.projectId = `project_id = ${escape(projectId)}`;
   const select = mergeDeepRight(
     {
@@ -380,6 +401,7 @@ export async function getEventList({
       name: true,
       deviceId: true,
       profileId: true,
+      sessionId: true,
       projectId: true,
       createdAt: true,
       path: true,
@@ -607,3 +629,320 @@ export async function getTopPages({
 
   return res;
 }
+
+export interface IEventServiceGetList {
+  projectId: string;
+  profileId?: string;
+  cursor?: Date;
+  filters?: IChartEventFilter[];
+}
+
+class EventService {
+  constructor(private client: typeof ch) {}
+
+  query<T>({
+    projectId,
+    profileId,
+    where,
+    select,
+    limit,
+    orderBy,
+  }: {
+    projectId: string;
+    profileId?: string;
+    where?: {
+      profile?: (query: Query<T>) => void;
+      event?: (query: Query<T>) => void;
+      session?: (query: Query<T>) => void;
+    };
+    select: {
+      profile?: Partial<SelectHelper<IServiceProfile>>;
+      event: Partial<SelectHelper<IServiceEvent>>;
+    };
+    limit?: number;
+    orderBy?: keyof IClickhouseEvent;
+  }) {
+    const events = clix(this.client)
+      .select<
+        Partial<IClickhouseEvent> & {
+          // profile
+          profileId: string;
+          profile_firstName: string;
+          profile_lastName: string;
+          profile_avatar: string;
+          profile_isExternal: boolean;
+          profile_createdAt: string;
+        }
+      >([
+        select.event.id && 'e.id as id',
+        select.event.deviceId && 'e.device_id as device_id',
+        select.event.name && 'e.name as name',
+        select.event.path && 'e.path as path',
+        select.event.duration && 'e.duration as duration',
+        select.event.country && 'e.country as country',
+        select.event.city && 'e.city as city',
+        select.event.os && 'e.os as os',
+        select.event.browser && 'e.browser as browser',
+        select.event.createdAt && 'e.created_at as created_at',
+        select.event.projectId && 'e.project_id as project_id',
+        'e.session_id as session_id',
+        'e.profile_id as profile_id',
+      ])
+      .from('events e')
+      .where('project_id', '=', projectId)
+      .when(!!where?.event, where?.event)
+      // Do not limit if profileId, we will limit later since we need the "correct" profileId
+      .when(!!limit && !profileId, (q) => q.limit(limit!))
+      .orderBy('toDate(created_at)', 'DESC')
+      .orderBy('created_at', 'DESC');
+
+    const sessions = clix(this.client)
+      .select(['id as session_id', 'profile_id'])
+      .from('sessions')
+      .where('sign', '=', 1)
+      .where('project_id', '=', projectId)
+      .when(!!where?.session, where?.session)
+      .when(!!profileId, (q) => q.where('profile_id', '=', profileId));
+
+    const profiles = clix(this.client)
+      .select([
+        'id',
+        'any(created_at) as created_at',
+        `any(nullIf(first_name, '')) as first_name`,
+        `any(nullIf(last_name, '')) as last_name`,
+        `any(nullIf(email, '')) as email`,
+        `any(nullIf(avatar, '')) as avatar`,
+        'last_value(is_external) as is_external',
+      ])
+      .from('profiles')
+      .where('project_id', '=', projectId)
+      .where(
+        'id',
+        'IN',
+        clix.exp(
+          clix(this.client)
+            .select(['profile_id'])
+            .from(
+              clix.exp(
+                clix(this.client)
+                  .select(['profile_id'])
+                  .from('cte_sessions')
+                  .union(
+                    clix(this.client).select(['profile_id']).from('cte_events'),
+                  ),
+              ),
+            )
+            .groupBy(['profile_id']),
+        ),
+      )
+      .groupBy(['id', 'project_id'])
+      .when(!!where?.profile, where?.profile);
+
+    return clix(this.client)
+      .with('cte_events', events)
+      .with('cte_sessions', sessions)
+      .with('cte_profiles', profiles)
+      .select<
+        Partial<IClickhouseEvent> & {
+          // profile
+          profileId: string;
+          profile_firstName: string;
+          profile_lastName: string;
+          profile_avatar: string;
+          profile_isExternal: boolean;
+          profile_createdAt: string;
+        }
+      >([
+        select.event.id && 'e.id as id',
+        select.event.deviceId && 'e.device_id as device_id',
+        select.event.name && 'e.name as name',
+        select.event.path && 'e.path as path',
+        select.event.duration && 'e.duration as duration',
+        select.event.country && 'e.country as country',
+        select.event.city && 'e.city as city',
+        select.event.os && 'e.os as os',
+        select.event.browser && 'e.browser as browser',
+        select.event.createdAt && 'e.created_at as created_at',
+        select.event.projectId && 'e.project_id as project_id',
+        select.event.sessionId && 'e.session_id as session_id',
+        select.event.profileId && 'e.profile_id as event_profile_id',
+        // Profile
+        select.profile?.id && 'p.id as profile_id',
+        select.profile?.firstName && 'p.first_name as profile_first_name',
+        select.profile?.lastName && 'p.last_name as profile_last_name',
+        select.profile?.avatar && 'p.avatar as profile_avatar',
+        select.profile?.isExternal && 'p.is_external as profile_is_external',
+        select.profile?.createdAt && 'p.created_at as profile_created_at',
+        select.profile?.email && 'p.email as profile_email',
+        select.profile?.properties && 'p.properties as profile_properties',
+      ])
+      .from('cte_events e')
+      .leftJoin('cte_sessions s', 'e.session_id = s.session_id')
+      .leftJoin(
+        'cte_profiles p',
+        's.profile_id = p.id AND p.is_external = true',
+      )
+      .when(!!profileId, (q) => {
+        q.where('s.profile_id', '=', profileId);
+        q.limit(limit!);
+      });
+  }
+
+  transformFromQuery(res: any[]) {
+    return res
+      .map((item) => {
+        return Object.entries(item).reduce(
+          (acc, [prop, val]) => {
+            if (prop === 'event_profile_id' && val) {
+              if (!item.profile_id) {
+                return assocPath(['profile', 'id'], val, acc);
+              }
+            }
+
+            if (
+              prop.startsWith('profile_') &&
+              !path(['profile', prop.replace('profile_', '')], acc)
+            ) {
+              return assocPath(
+                ['profile', prop.replace('profile_', '')],
+                val,
+                acc,
+              );
+            }
+            return assocPath([prop], val, acc);
+          },
+          {
+            profile: {},
+          } as IClickhouseEvent,
+        );
+      })
+      .map(transformEvent);
+  }
+
+  async getById({
+    projectId,
+    id,
+    createdAt,
+  }: {
+    projectId: string;
+    id: string;
+    createdAt?: Date;
+  }) {
+    return clix(this.client)
+      .select<IClickhouseEvent>(['*'])
+      .from('events')
+      .where('project_id', '=', projectId)
+      .when(!!createdAt, (q) => {
+        if (createdAt) {
+          q.where('created_at', 'BETWEEN', [
+            new Date(createdAt.getTime() - 1000),
+            new Date(createdAt.getTime() + 1000),
+          ]);
+        }
+      })
+      .where('id', '=', id)
+      .limit(1)
+      .execute()
+      .then((res) => {
+        if (!res[0]) {
+          return null;
+        }
+
+        return transformEvent(res[0]);
+      });
+  }
+
+  async getList({
+    projectId,
+    profileId,
+    cursor,
+    filters,
+    limit = 50,
+    startDate,
+    endDate,
+  }: IEventServiceGetList & {
+    limit?: number;
+    startDate?: Date;
+    endDate?: Date;
+  }) {
+    const date = cursor || new Date();
+    const query = this.query({
+      projectId,
+      profileId,
+      limit,
+      orderBy: 'created_at',
+      select: {
+        event: {
+          deviceId: true,
+          profileId: true,
+          id: true,
+          name: true,
+          createdAt: true,
+          duration: true,
+          country: true,
+          city: true,
+          os: true,
+          browser: true,
+          path: true,
+          sessionId: true,
+        },
+        profile: {
+          id: true,
+          firstName: true,
+          lastName: true,
+          avatar: true,
+          isExternal: true,
+        },
+      },
+      where: {
+        event: (q) => {
+          if (startDate && endDate) {
+            q.where('created_at', 'BETWEEN', [
+              startDate ?? new Date(date.getTime() - 1000 * 60 * 60 * 24 * 3.5),
+              cursor ?? endDate,
+            ]);
+          } else {
+            q.where('created_at', '<', date);
+          }
+          if (filters) {
+            q.rawWhere(
+              Object.values(getEventFiltersWhereClause(filters)).join(' AND '),
+            );
+          }
+        },
+        session: (q) => {
+          if (startDate && endDate) {
+            q.where('created_at', 'BETWEEN', [
+              startDate ?? new Date(date.getTime() - 1000 * 60 * 60 * 24 * 3.5),
+              endDate ?? date,
+            ]);
+          } else {
+            q.where('created_at', '<', date);
+          }
+        },
+      },
+    })
+      .orderBy('toDate(created_at)', 'DESC')
+      .orderBy('created_at', 'DESC');
+
+    const results = await query.execute();
+
+    // Current page items (middle chunk)
+    const items = results.slice(0, limit);
+
+    // Check if there's a next page
+    const hasNext = results.length >= limit;
+
+    return {
+      items: this.transformFromQuery(items).map((item) => ({
+        ...item,
+        projectId: projectId,
+      })),
+      meta: {
+        next: hasNext ? last(items)?.created_at : null,
+      },
+    };
+  }
+}
+
+export const eventService = new EventService(ch);

packages/db/src/services/overview.service.ts

@@ -0,0 +1,639 @@
+import { average, sum } from '@openpanel/common';
+import { getCache } from '@openpanel/redis';
+import { type IChartEventFilter, zTimeInterval } from '@openpanel/validation';
+import { omit } from 'ramda';
+import { z } from 'zod';
+import { TABLE_NAMES, ch } from '../clickhouse/client';
+import { clix } from '../clickhouse/query-builder';
+import { getEventFiltersWhereClause } from './chart.service';
+
+export const zGetMetricsInput = z.object({
+  projectId: z.string(),
+  filters: z.array(z.any()),
+  startDate: z.string(),
+  endDate: z.string(),
+  interval: zTimeInterval,
+});
+
+export type IGetMetricsInput = z.infer<typeof zGetMetricsInput>;
+
+export const zGetTopPagesInput = z.object({
+  projectId: z.string(),
+  filters: z.array(z.any()),
+  startDate: z.string(),
+  endDate: z.string(),
+  interval: zTimeInterval,
+  cursor: z.number().optional(),
+  limit: z.number().optional(),
+});
+
+export type IGetTopPagesInput = z.infer<typeof zGetTopPagesInput>;
+
+export const zGetTopEntryExitInput = z.object({
+  projectId: z.string(),
+  filters: z.array(z.any()),
+  startDate: z.string(),
+  endDate: z.string(),
+  interval: zTimeInterval,
+  mode: z.enum(['entry', 'exit']),
+  cursor: z.number().optional(),
+  limit: z.number().optional(),
+});
+
+export type IGetTopEntryExitInput = z.infer<typeof zGetTopEntryExitInput>;
+
+export const zGetTopGenericInput = z.object({
+  projectId: z.string(),
+  filters: z.array(z.any()),
+  startDate: z.string(),
+  endDate: z.string(),
+  interval: zTimeInterval,
+  column: z.enum([
+    // Referrers
+    'referrer',
+    'referrer_name',
+    'referrer_type',
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_term',
+    'utm_content',
+    // Geo
+    'region',
+    'country',
+    'city',
+    // Device
+    'device',
+    'brand',
+    'model',
+    'browser',
+    'browser_version',
+    'os',
+    'os_version',
+  ]),
+  cursor: z.number().optional(),
+  limit: z.number().optional(),
+});
+
+export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput>;
+
+export class OverviewService {
+  private pendingQueries: Map<string, Promise<number | null>> = new Map();
+
+  constructor(private client: typeof ch) {}
+
+  isPageFilter(filters: IChartEventFilter[]) {
+    return filters.some((filter) => filter.name === 'path' && filter.value);
+  }
+
+  getTotalSessions({
+    projectId,
+    startDate,
+    endDate,
+    filters,
+  }: {
+    projectId: string;
+    startDate: string;
+    endDate: string;
+    filters: IChartEventFilter[];
+  }) {
+    const where = this.getRawWhereClause('sessions', filters);
+    const key = `total_sessions_${projectId}_${startDate}_${endDate}_${JSON.stringify(filters)}`;
+
+    // Check if there's already a pending query for this key
+    const pendingQuery = this.pendingQueries.get(key);
+    if (pendingQuery) {
+      return pendingQuery.then((res) => res ?? 0);
+    }
+
+    // Create new query promise and store it
+    const queryPromise = getCache(key, 15, async () => {
+      try {
+        const result = await clix(this.client)
+          .select<{
+            total_sessions: number;
+          }>(['sum(sign) as total_sessions'])
+          .from(TABLE_NAMES.sessions, true)
+          .where('project_id', '=', projectId)
+          .where('created_at', 'BETWEEN', [
+            clix.datetime(startDate),
+            clix.datetime(endDate),
+          ])
+          .rawWhere(where)
+          .having('sum(sign)', '>', 0)
+          .execute();
+        return result?.[0]?.total_sessions ?? 0;
+      } catch (error) {
+        return 0;
+      }
+    });
+
+    this.pendingQueries.set(key, queryPromise);
+    return queryPromise;
+  }
+
+  getMetrics({
+    projectId,
+    filters,
+    startDate,
+    endDate,
+    interval,
+  }: IGetMetricsInput): Promise<{
+    metrics: {
+      bounce_rate: number;
+      unique_visitors: number;
+      total_sessions: number;
+      avg_session_duration: number;
+      total_screen_views: number;
+      views_per_session: number;
+    };
+    series: {
+      date: string;
+      bounce_rate: number;
+      unique_visitors: number;
+      total_sessions: number;
+      avg_session_duration: number;
+      total_screen_views: number;
+      views_per_session: number;
+    }[];
+  }> {
+    const where = this.getRawWhereClause('sessions', filters);
+    if (this.isPageFilter(filters)) {
+      // Session aggregation with bounce rates
+      const sessionAggQuery = clix(this.client)
+        .select([
+          `${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
+          'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
+        ])
+        .from(TABLE_NAMES.sessions, true)
+        .where('sign', '=', 1)
+        .where('project_id', '=', projectId)
+        .where('created_at', 'BETWEEN', [
+          clix.datetime(startDate),
+          clix.datetime(endDate),
+        ])
+        .rawWhere(where)
+        .groupBy(['date'])
+        .rollup()
+        .orderBy('date', 'ASC');
+
+      // Overall unique visitors
+      const overallUniqueVisitorsQuery = clix(this.client)
+        .select([
+          'uniq(profile_id) AS unique_visitors',
+          'uniq(session_id) AS total_sessions',
+        ])
+        .from(TABLE_NAMES.events)
+        .where('project_id', '=', projectId)
+        .where('name', '=', 'screen_view')
+        .where('created_at', 'BETWEEN', [
+          clix.datetime(startDate),
+          clix.datetime(endDate),
+        ])
+        .rawWhere(this.getRawWhereClause('events', filters));
+
+      return clix(this.client)
+        .with('session_agg', sessionAggQuery)
+        .with(
+          'overall_bounce_rate',
+          clix(this.client)
+            .select(['bounce_rate'])
+            .from('session_agg')
+            .where('date', '=', clix.exp("'1970-01-01 00:00:00'")),
+        )
+        .with(
+          'daily_stats',
+          clix(this.client)
+            .select(['date', 'bounce_rate'])
+            .from('session_agg')
+            .where('date', '!=', clix.exp("'1970-01-01 00:00:00'")),
+        )
+        .with('overall_unique_visitors', overallUniqueVisitorsQuery)
+        .select<{
+          date: string;
+          bounce_rate: number;
+          unique_visitors: number;
+          total_sessions: number;
+          avg_session_duration: number;
+          total_screen_views: number;
+          views_per_session: number;
+          overall_unique_visitors: number;
+          overall_total_sessions: number;
+          overall_bounce_rate: number;
+        }>([
+          `${clix.toStartOfInterval('e.created_at', interval, startDate)} AS date`,
+          'ds.bounce_rate as bounce_rate',
+          'uniq(e.profile_id) AS unique_visitors',
+          'uniq(e.session_id) AS total_sessions',
+          'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
+          'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
+          'count(*) AS total_screen_views',
+          'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
+          '(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
+          '(SELECT total_sessions FROM overall_unique_visitors) AS overall_total_sessions',
+          '(SELECT bounce_rate FROM overall_bounce_rate) AS overall_bounce_rate',
+        ])
+        .from(`${TABLE_NAMES.events} AS e`)
+        .leftJoin(
+          'daily_stats AS ds',
+          `${clix.toStartOfInterval('e.created_at', interval, startDate)} = ds.date`,
+        )
+        .where('e.project_id', '=', projectId)
+        .where('e.name', '=', 'screen_view')
+        .where('e.created_at', 'BETWEEN', [
+          clix.datetime(startDate),
+          clix.datetime(endDate),
+        ])
+        .rawWhere(this.getRawWhereClause('events', filters))
+        .groupBy(['date', 'ds.bounce_rate'])
+        .orderBy('date', 'ASC')
+        .fill(
+          clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
+          clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
+          clix.toInterval('1', interval),
+        )
+        .transform({
+          date: (item) => new Date(item.date).toISOString(),
+        })
+        .execute()
+        .then((res) => {
+          const anyRowWithData = res.find(
+            (item) =>
+              item.overall_bounce_rate !== null ||
+              item.overall_total_sessions !== null ||
+              item.overall_unique_visitors !== null,
+          );
+          return {
+            metrics: {
+              bounce_rate: anyRowWithData?.overall_bounce_rate ?? 0,
+              unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0,
+              total_sessions: anyRowWithData?.overall_total_sessions ?? 0,
+              avg_session_duration: average(
+                res.map((item) => item.avg_session_duration),
+              ),
+              total_screen_views: sum(
+                res.map((item) => item.total_screen_views),
+              ),
+              views_per_session: average(
+                res.map((item) => item.views_per_session),
+              ),
+            },
+            series: res.map(
+              omit([
+                'overall_bounce_rate',
+                'overall_unique_visitors',
+                'overall_total_sessions',
+              ]),
+            ),
+          };
+        });
+    }
+
+    const query = clix(this.client)
+      .select<{
+        date: string;
+        bounce_rate: number;
+        unique_visitors: number;
+        total_sessions: number;
+        avg_session_duration: number;
+        total_screen_views: number;
+        views_per_session: number;
+      }>([
+        `${clix.toStartOfInterval('created_at', interval, startDate)} AS date`,
+        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
+        'uniqIf(profile_id, sign > 0) AS unique_visitors',
+        'sum(sign) AS total_sessions',
+        'round(avgIf(duration, duration > 0 AND sign > 0), 2) / 1000 AS _avg_session_duration',
+        'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
+        'sum(sign * screen_view_count) AS total_screen_views',
+        'round(sum(sign * screen_view_count) * 1.0 / sum(sign), 2) AS views_per_session',
+      ])
+      .from('sessions')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate),
+        clix.datetime(endDate),
+      ])
+      .where('project_id', '=', projectId)
+      .rawWhere(where)
+      .groupBy(['date'])
+      .having('sum(sign)', '>', 0)
+      .rollup()
+      .orderBy('date', 'ASC')
+      .fill(
+        clix.toStartOfInterval(clix.datetime(startDate), interval, startDate),
+        clix.toStartOfInterval(clix.datetime(endDate), interval, startDate),
+        clix.toInterval('1', interval),
+      )
+      .transform({
+        date: (item) => new Date(item.date).toISOString(),
+      });
+
+    return query.execute().then((res) => {
+      // First row is the rollup row containing the total values
+      return {
+        metrics: {
+          bounce_rate: res[0]?.bounce_rate ?? 0,
+          unique_visitors: res[0]?.unique_visitors ?? 0,
+          total_sessions: res[0]?.total_sessions ?? 0,
+          avg_session_duration: res[0]?.avg_session_duration ?? 0,
+          total_screen_views: res[0]?.total_screen_views ?? 0,
+          views_per_session: res[0]?.views_per_session ?? 0,
+        },
+        series: res
+          .slice(1)
+          .map(omit(['overall_bounce_rate', 'overall_unique_visitors'])),
+      };
+    });
+  }
+
+  getRawWhereClause(type: 'events' | 'sessions', filters: IChartEventFilter[]) {
+    const where = getEventFiltersWhereClause(
+      filters.map((item) => {
+        if (type === 'sessions') {
+          if (item.name === 'path') {
+            return { ...item, name: 'entry_path' };
+          }
+          if (item.name === 'origin') {
+            return { ...item, name: 'entry_origin' };
+          }
+          if (item.name.startsWith('properties.__query.utm_')) {
+            return {
+              ...item,
+              name: item.name.replace('properties.__query.utm_', 'utm_'),
+            };
+          }
+          return item;
+        }
+        return item;
+      }),
+      // .filter((item) => {
+      //   if (this.isPageFilter(filters) && type === 'sessions') {
+      //     return item.name !== 'entry_path' && item.name !== 'entry_origin';
+      //   }
+      //   return true;
+      // }),
+    );
+
+    return Object.values(where).join(' AND ');
+  }
+
+  async getTopPages({
+    projectId,
+    filters,
+    startDate,
+    endDate,
+    cursor = 1,
+    limit = 10,
+  }: IGetTopPagesInput) {
+    const pageStatsQuery = clix(this.client)
+      .select([
+        'origin',
+        'path',
+        'uniq(session_id) as count',
+        'round(avg(duration)/1000, 2) as avg_duration',
+      ])
+      .from(TABLE_NAMES.events, false)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'screen_view')
+      .where('path', '!=', '')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate),
+        clix.datetime(endDate),
+      ])
+      .groupBy(['origin', 'path'])
+      .orderBy('count', 'DESC')
+      .limit(limit)
+      .offset((cursor - 1) * limit);
+
+    const bounceStatsQuery = clix(this.client)
+      .select([
+        'entry_path',
+        'entry_origin',
+        'coalesce(round(countIf(is_bounce = 1 AND sign = 1) * 100.0 / countIf(sign = 1), 2), 0) as bounce_rate',
+      ])
+      .from(TABLE_NAMES.sessions, true)
+      .where('sign', '=', 1)
+      .where('project_id', '=', projectId)
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate),
+        clix.datetime(endDate),
+      ])
+      .groupBy(['entry_path', 'entry_origin']);
+
+    pageStatsQuery.rawWhere(this.getRawWhereClause('events', filters));
+    bounceStatsQuery.rawWhere(this.getRawWhereClause('sessions', filters));
+
+    const mainQuery = clix(this.client)
+      .with('page_stats', pageStatsQuery)
+      .with('bounce_stats', bounceStatsQuery)
+      .select<{
+        origin: string;
+        path: string;
+        avg_duration: number;
+        bounce_rate: number;
+        sessions: number;
+      }>([
+        'p.origin',
+        'p.path',
+        'p.avg_duration',
+        'p.count as sessions',
+        'b.bounce_rate',
+      ])
+      .from('page_stats p', false)
+      .leftJoin(
+        'bounce_stats b',
+        'p.path = b.entry_path AND p.origin = b.entry_origin',
+      )
+      .orderBy('sessions', 'DESC')
+      .limit(limit);
+
+    const totalSessions = await this.getTotalSessions({
+      projectId,
+      startDate,
+      endDate,
+      filters,
+    });
+
+    return mainQuery.execute();
+  }
+
+  async getTopEntryExit({
+    projectId,
+    filters,
+    startDate,
+    endDate,
+    mode,
+    cursor = 1,
+    limit = 10,
+  }: IGetTopEntryExitInput) {
+    const where = this.getRawWhereClause('sessions', filters);
+
+    const distinctSessionQuery = this.getDistinctSessions({
+      projectId,
+      filters,
+      startDate,
+      endDate,
+    });
+
+    const offset = (cursor - 1) * limit;
+
+    const query = clix(this.client)
+      .select<{
+        origin: string;
+        path: string;
+        avg_duration: number;
+        bounce_rate: number;
+        sessions: number;
+      }>([
+        `${mode}_origin AS origin`,
+        `${mode}_path AS path`,
+        'round(avg(duration * sign)/1000, 2) as avg_duration',
+        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
+        'sum(sign) as sessions',
+      ])
+      .from(TABLE_NAMES.sessions, true)
+      .where('project_id', '=', projectId)
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate),
+        clix.datetime(endDate),
+      ])
+      .rawWhere(where)
+      .groupBy([`${mode}_origin`, `${mode}_path`])
+      .having('sum(sign)', '>', 0)
+      .orderBy('sessions', 'DESC')
+      .limit(limit)
+      .offset(offset);
+
+    let mainQuery = query;
+
+    if (this.isPageFilter(filters)) {
+      mainQuery = clix(this.client)
+        .with('distinct_sessions', distinctSessionQuery)
+        .merge(query)
+        .where(
+          'id',
+          'IN',
+          clix.exp('(SELECT session_id FROM distinct_sessions)'),
+        );
+    }
+
+    const totalSessions = await this.getTotalSessions({
+      projectId,
+      startDate,
+      endDate,
+      filters,
+    });
+
+    return mainQuery.execute();
+  }
+
+  private getDistinctSessions({
+    projectId,
+    filters,
+    startDate,
+    endDate,
+  }: {
+    projectId: string;
+    filters: IChartEventFilter[];
+    startDate: string;
+    endDate: string;
+  }) {
+    return clix(this.client)
+      .select(['DISTINCT session_id'])
+      .from(TABLE_NAMES.events)
+      .where('project_id', '=', projectId)
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate),
+        clix.datetime(endDate),
+      ])
+      .rawWhere(this.getRawWhereClause('events', filters));
+  }
+
+  async getTopGeneric({
+    projectId,
+    filters,
+    startDate,
+    endDate,
+    column,
+    cursor = 1,
+    limit = 10,
+  }: IGetTopGenericInput) {
+    const distinctSessionQuery = this.getDistinctSessions({
+      projectId,
+      filters,
+      startDate,
+      endDate,
+    });
+
+    const prefixColumn = (() => {
+      switch (column) {
+        case 'region':
+          return 'country';
+        case 'city':
+          return 'country';
+        case 'browser_version':
+          return 'browser';
+        case 'os_version':
+          return 'os';
+      }
+      return null;
+    })();
+
+    const offset = (cursor - 1) * limit;
+
+    const query = clix(this.client)
+      .select<{
+        prefix?: string;
+        name: string;
+        sessions: number;
+        bounce_rate: number;
+        avg_session_duration: number;
+      }>([
+        prefixColumn && `${prefixColumn} as prefix`,
+        `nullIf(${column}, '') as name`,
+        'sum(sign) as sessions',
+        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) AS bounce_rate',
+        'round(avgIf(duration, duration > 0 AND sign > 0), 2)/1000 AS avg_session_duration',
+      ])
+      .from(TABLE_NAMES.sessions, true)
+      .where('project_id', '=', projectId)
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate),
+        clix.datetime(endDate),
+      ])
+      .groupBy([prefixColumn, column].filter(Boolean))
+      .having('sum(sign)', '>', 0)
+      .orderBy('sessions', 'DESC')
+      .limit(limit)
+      .offset(offset);
+
+    let mainQuery = query;
+
+    if (this.isPageFilter(filters)) {
+      mainQuery = clix(this.client)
+        .with('distinct_sessions', distinctSessionQuery)
+        .merge(query)
+        .where(
+          'id',
+          'IN',
+          clix.exp('(SELECT session_id FROM distinct_sessions)'),
+        );
+    } else {
+      mainQuery.rawWhere(this.getRawWhereClause('sessions', filters));
+    }
+
+    const [res, totalSessions] = await Promise.all([
+      mainQuery.execute(),
+      this.getTotalSessions({
+        projectId,
+        startDate,
+        endDate,
+        filters,
+      }),
+    ]);
+
+    return res;
+  }
+}
+
+export const overviewService = new OverviewService(ch);

packages/db/src/services/profile.service.ts

@@ -49,7 +49,17 @@ export async function getProfileById(id: string, projectId: string) {
   }
 
   const [profile] = await chQuery<IClickhouseProfile>(
-    `SELECT * FROM ${TABLE_NAMES.profiles} WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`,
+    `SELECT 
+      id, 
+      project_id,
+      last_value(nullIf(first_name, '')) as first_name, 
+      last_value(nullIf(last_name, '')) as last_name, 
+      last_value(nullIf(email, '')) as email, 
+      last_value(nullIf(avatar, '')) as avatar, 
+      last_value(is_external) as is_external, 
+      last_value(properties) as properties, 
+      last_value(created_at) as created_at
+    FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
   );
 
   if (!profile) {
@@ -59,7 +69,7 @@ export async function getProfileById(id: string, projectId: string) {
   return transformProfile(profile);
 }
 
-export const getProfileByIdCached = cacheable(getProfileById, 60 * 30);
+export const getProfileByIdCached = getProfileById; //cacheable(getProfileById, 60 * 30);
 
 interface GetProfileListOptions {
   projectId: string;
@@ -77,11 +87,21 @@ export async function getProfiles(ids: string[], projectId: string) {
   }
 
   const data = await chQuery<IClickhouseProfile>(
-    `SELECT id, first_name, last_name, email, avatar, is_external, properties, created_at
+    `SELECT 
-    FROM ${TABLE_NAMES.profiles} FINAL 
+      id, 
+      project_id,
+      any(nullIf(first_name, '')) as first_name, 
+      any(nullIf(last_name, '')) as last_name, 
+      any(nullIf(email, '')) as email, 
+      any(nullIf(avatar, '')) as avatar, 
+      last_value(is_external) as is_external, 
+      any(properties) as properties, 
+      any(created_at) as created_at
+    FROM ${TABLE_NAMES.profiles}
     WHERE 
       project_id = ${escape(projectId)} AND
       id IN (${filteredIds.map((id) => escape(id)).join(',')})
+    GROUP BY id, project_id
     `,
   );
 

packages/db/src/services/session.service.ts

@@ -0,0 +1,41 @@
+export type IClickhouseSession = {
+  id: string;
+  profile_id: string;
+  event_count: number;
+  screen_view_count: number;
+  screen_views: string[];
+  entry_path: string;
+  entry_origin: string;
+  exit_path: string;
+  exit_origin: string;
+  created_at: string;
+  ended_at: string;
+  referrer: string;
+  referrer_name: string;
+  referrer_type: string;
+  os: string;
+  os_version: string;
+  browser: string;
+  browser_version: string;
+  device: string;
+  brand: string;
+  model: string;
+  country: string;
+  region: string;
+  city: string;
+  longitude: number | null;
+  latitude: number | null;
+  is_bounce: boolean;
+  project_id: string;
+  device_id: string;
+  duration: number;
+  utm_medium: string;
+  utm_source: string;
+  utm_campaign: string;
+  utm_content: string;
+  utm_term: string;
+  revenue: number;
+  sign: 1 | 0;
+  version: number;
+  properties: Record<string, string>;
+};

packages/db/src/sql-builder.ts

@@ -7,6 +7,7 @@ export interface SqlBuilderObject {
   groupBy: Record<string, string>;
   orderBy: Record<string, string>;
   from: string;
+  joins: Record<string, string>;
   limit: number | undefined;
   offset: number | undefined;
 }
@@ -17,11 +18,12 @@ export function createSqlBuilder() {
 
   const sb: SqlBuilderObject = {
     where: {},
-    from: TABLE_NAMES.events,
+    from: `${TABLE_NAMES.events} e`,
     select: {},
     groupBy: {},
     orderBy: {},
     having: {},
+    joins: {},
     limit: undefined,
     offset: undefined,
   };
@@ -39,6 +41,8 @@ export function createSqlBuilder() {
     Object.keys(sb.orderBy).length ? `ORDER BY ${join(sb.orderBy, ', ')}` : '';
   const getLimit = () => (sb.limit ? `LIMIT ${sb.limit}` : '');
   const getOffset = () => (sb.offset ? `OFFSET ${sb.offset}` : '');
+  const getJoins = () =>
+    Object.keys(sb.joins).length ? join(sb.joins, ' ') : '';
 
   return {
     sb,
@@ -49,10 +53,12 @@ export function createSqlBuilder() {
     getGroupBy,
     getOrderBy,
     getHaving,
+    getJoins,
     getSql: () => {
       const sql = [
         getSelect(),
         getFrom(),
+        getJoins(),
         getWhere(),
         getGroupBy(),
         getHaving(),

packages/email/src/index.tsx

@@ -17,7 +17,7 @@ export async function sendEmail<T extends TemplateKey>(
   const { subject, Component, schema } = templates[template];
   const props = schema.safeParse(data);
 
-  if (props.error) {
+  if (!props.success) {
     console.error('Failed to parse data', props.error);
     return null;
   }

packages/logger/index.ts

@@ -85,6 +85,7 @@ export function createLogger({ name }: { name: string }): ILogger {
     level: logLevel,
     format,
     transports,
+    // silent: true,
     // Add ISO levels of logging from PINO
     levels: Object.assign(
       { fatal: 0, warn: 4, trace: 7 },

packages/queue/src/queues.ts

@@ -56,6 +56,10 @@ export type CronQueuePayloadFlushProfiles = {
   type: 'flushProfiles';
   payload: undefined;
 };
+export type CronQueuePayloadFlushSessions = {
+  type: 'flushSessions';
+  payload: undefined;
+};
 export type CronQueuePayloadPing = {
   type: 'ping';
   payload: undefined;
@@ -67,6 +71,7 @@ export type CronQueuePayloadProject = {
 export type CronQueuePayload =
   | CronQueuePayloadSalt
   | CronQueuePayloadFlushEvents
+  | CronQueuePayloadFlushSessions
   | CronQueuePayloadFlushProfiles
   | CronQueuePayloadPing
   | CronQueuePayloadProject;

packages/redis/redis.ts

@@ -1,3 +1,4 @@
+import { getSuperJson, setSuperJson } from '@openpanel/json';
 import type { RedisOptions } from 'ioredis';
 import Redis from 'ioredis';
 
@@ -7,20 +8,59 @@ const options: RedisOptions = {
 
 export { Redis };
 
+export interface ExtendedRedis extends Redis {
+  getJson: <T = any>(key: string) => Promise<T | null>;
+  setJson: <T = any>(
+    key: string,
+    expireInSec: number,
+    value: T,
+  ) => Promise<void>;
+}
+
 const createRedisClient = (
   url: string,
   overrides: RedisOptions = {},
-): Redis => {
+): ExtendedRedis => {
-  const client = new Redis(url, { ...options, ...overrides });
+  const client = new Redis(url, {
+    ...options,
+    ...overrides,
+  }) as ExtendedRedis;
 
   client.on('error', (error) => {
     console.error('Redis Client Error:', error);
   });
 
+  client.getJson = async <T = any>(key: string): Promise<T | null> => {
+    const value = await client.get(key);
+    if (value) {
+      const res = getSuperJson(value) as T;
+      if (res && Array.isArray(res) && res.length === 0) {
+        return null;
+      }
+
+      if (res && typeof res === 'object' && Object.keys(res).length === 0) {
+        return null;
+      }
+
+      if (res) {
+        return res;
+      }
+    }
+    return null;
+  };
+
+  client.setJson = async <T = any>(
+    key: string,
+    expireInSec: number,
+    value: T,
+  ): Promise<void> => {
+    await client.setex(key, expireInSec, setSuperJson(value));
+  };
+
   return client;
 };
 
-let redisCache: Redis;
+let redisCache: ExtendedRedis;
 export function getRedisCache() {
   if (!redisCache) {
     redisCache = createRedisClient(process.env.REDIS_URL!, options);
@@ -29,7 +69,7 @@ export function getRedisCache() {
   return redisCache;
 }
 
-let redisSub: Redis;
+let redisSub: ExtendedRedis;
 export function getRedisSub() {
   if (!redisSub) {
     redisSub = createRedisClient(process.env.REDIS_URL!, options);
@@ -38,7 +78,7 @@ export function getRedisSub() {
   return redisSub;
 }
 
-let redisPub: Redis;
+let redisPub: ExtendedRedis;
 export function getRedisPub() {
   if (!redisPub) {
     redisPub = createRedisClient(process.env.REDIS_URL!, options);
@@ -47,7 +87,7 @@ export function getRedisPub() {
   return redisPub;
 }
 
-let redisQueue: Redis;
+let redisQueue: ExtendedRedis;
 export function getRedisQueue() {
   if (!redisQueue) {
     // Use different redis for queues (self-hosting will re-use the same redis instance)

packages/trpc/src/root.ts

@@ -7,6 +7,7 @@ import { integrationRouter } from './routers/integration';
 import { notificationRouter } from './routers/notification';
 import { onboardingRouter } from './routers/onboarding';
 import { organizationRouter } from './routers/organization';
+import { overviewRouter } from './routers/overview';
 import { profileRouter } from './routers/profile';
 import { projectRouter } from './routers/project';
 import { referenceRouter } from './routers/reference';
@@ -39,6 +40,7 @@ export const appRouter = createTRPCRouter({
   integration: integrationRouter,
   auth: authRouter,
   subscription: subscriptionRouter,
+  overview: overviewRouter,
 });
 
 // export type definition of API

packages/trpc/src/routers/chart.helpers.ts

@@ -291,12 +291,11 @@ export function getChartPrevStartEndDate({
 }: {
   startDate: string;
   endDate: string;
-  range: IChartRange;
 }) {
   const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate));
   return {
-    startDate: formatISO(subMilliseconds(new Date(startDate), diff - 1)),
+    startDate: formatISO(subMilliseconds(new Date(startDate), diff + 1000)),
-    endDate: formatISO(subMilliseconds(new Date(endDate), diff - 1)),
+    endDate: formatISO(subMilliseconds(new Date(endDate), diff + 1000)),
   };
 }
 
@@ -307,7 +306,10 @@ export async function getFunnelData({
   ...payload
 }: IChartInput) {
   const funnelWindow = (payload.funnelWindow || 24) * 3600;
-  const funnelGroup = payload.funnelGroup || 'session_id';
+  const funnelGroup =
+    payload.funnelGroup === 'profile_id'
+      ? [`COALESCE(nullIf(s.profile_id, ''), e.profile_id)`, 'profile_id']
+      : ['session_id', 'session_id'];
 
   if (!startDate || !endDate) {
     throw new Error('startDate and endDate are required');
@@ -327,16 +329,19 @@ export async function getFunnelData({
     return getWhere().replace('WHERE ', '');
   });
 
-  const innerSql = `SELECT
+  const commonWhere = `project_id = ${escape(projectId)} AND 
-    ${funnelGroup},
-    windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
-  FROM ${TABLE_NAMES.events}
-  WHERE 
-    project_id = ${escape(projectId)} AND 
     created_at >= '${formatClickhouseDate(startDate)}' AND 
-    created_at <= '${formatClickhouseDate(endDate)}' AND
+    created_at <= '${formatClickhouseDate(endDate)}'`;
+
+  const innerSql = `SELECT
+    ${funnelGroup[0]} AS ${funnelGroup[1]},
+    windowFunnel(${funnelWindow}, 'strict_increase')(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
+  FROM ${TABLE_NAMES.events} e
+  ${funnelGroup[0] === 'session_id' ? '' : `LEFT JOIN (SELECT profile_id, id FROM sessions WHERE ${commonWhere}) AS s ON s.id = e.session_id`}
+  WHERE 
+    ${commonWhere} AND
     name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
-  GROUP BY ${funnelGroup}`;
+  GROUP BY ${funnelGroup[0]}`;
 
   const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;
 
@@ -513,10 +518,7 @@ export async function getChart(input: IChartInput) {
   }
 
   const currentPeriod = getChartStartEndDate(input);
-  const previousPeriod = getChartPrevStartEndDate({
+  const previousPeriod = getChartPrevStartEndDate(currentPeriod);
-    range: input.range,
-    ...currentPeriod,
-  });
 
   // If the current period end date is after the subscription chart end date, we need to use the subscription chart end date
   if (

packages/trpc/src/routers/chart.ts

@@ -181,10 +181,7 @@ export const chartRouter = createTRPCRouter({
 
   funnel: protectedProcedure.input(zChartInput).query(async ({ input }) => {
     const currentPeriod = getChartStartEndDate(input);
-    const previousPeriod = getChartPrevStartEndDate({
+    const previousPeriod = getChartPrevStartEndDate(currentPeriod);
-      range: input.range,
-      ...currentPeriod,
-    });
 
     const [current, previous] = await Promise.all([
       getFunnelData({ ...input, ...currentPeriod }),

packages/trpc/src/routers/event.ts

@@ -3,16 +3,21 @@ import { escape } from 'sqlstring';
 import { z } from 'zod';
 
 import {
+  type IServiceProfile,
   TABLE_NAMES,
   chQuery,
   convertClickhouseDateToJs,
   db,
+  eventService,
+  formatClickhouseDate,
   getEventList,
   getEvents,
   getTopPages,
 } from '@openpanel/db';
 import { zChartEventFilter } from '@openpanel/validation';
 
+import { addMinutes, subMinutes } from 'date-fns';
+import { clone } from 'ramda';
 import { getProjectAccessCached } from '../access';
 import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
@@ -48,51 +53,83 @@ export const eventRouter = createTRPCRouter({
       z.object({
         id: z.string(),
         projectId: z.string(),
+        createdAt: z.date().optional(),
       }),
     )
-    .query(async ({ input: { id, projectId } }) => {
+    .query(async ({ input: { id, projectId, createdAt } }) => {
-      const res = await getEvents(
+      const res = await eventService.getById({
-        `SELECT * FROM ${TABLE_NAMES.events} WHERE id = ${escape(id)} AND project_id = ${escape(projectId)};`,
+        projectId,
-        {
+        id,
-          meta: true,
+        createdAt,
-        },
+      });
-      );
 
-      if (!res?.[0]) {
+      if (!res) {
         throw new TRPCError({
           code: 'NOT_FOUND',
           message: 'Event not found',
         });
       }
 
-      return res[0];
+      return res;
     }),
 
   events: protectedProcedure
     .input(
       z.object({
         projectId: z.string(),
-        cursor: z.number().optional(),
         profileId: z.string().optional(),
-        take: z.number().default(50),
+        cursor: z.string().optional(),
-        events: z.array(z.string()).optional(),
         filters: z.array(zChartEventFilter).default([]),
         startDate: z.date().optional(),
         endDate: z.date().optional(),
-        meta: z.boolean().optional(),
-        profile: z.boolean().optional(),
       }),
     )
     .query(async ({ input }) => {
-      return getEventList(input);
+      const items = await getEventList({
+        ...input,
+        take: 50,
+        cursor: input.cursor ? new Date(input.cursor) : undefined,
+      });
+
+      // Hacky join to get profile for entire session
+      // TODO: Replace this with a join on the session table
+      const map = new Map<string, IServiceProfile>(); // sessionId -> profileId
+      for (const item of items) {
+        if (item.sessionId && item.profile?.isExternal === true) {
+          map.set(item.sessionId, item.profile);
+        }
+      }
+
+      for (const item of items) {
+        const profile = map.get(item.sessionId);
+        if (profile && (item.profile?.isExternal === false || !item.profile)) {
+          item.profile = clone(profile);
+          if (item?.profile?.firstName) {
+            item.profile.firstName = `* ${item.profile.firstName}`;
+          }
+        }
+      }
+
+      const lastItem = items[items.length - 1];
+
+      return {
+        items,
+        meta: {
+          next:
+            items.length === 50 && lastItem
+              ? lastItem.createdAt.toISOString()
+              : null,
+        },
+      };
     }),
   conversions: protectedProcedure
     .input(
       z.object({
         projectId: z.string(),
+        cursor: z.string().optional(),
       }),
     )
-    .query(async ({ input: { projectId } }) => {
+    .query(async ({ input: { projectId, cursor } }) => {
       const conversions = await db.eventMeta.findMany({
         where: {
           projectId,
@@ -101,16 +138,30 @@ export const eventRouter = createTRPCRouter({
       });
 
       if (conversions.length === 0) {
-        return [];
+        return {
+          items: [],
+          meta: {
+            next: null,
+          },
+        };
       }
 
-      return getEvents(
+      const items = await getEvents(
-        `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,
+        `SELECT * FROM ${TABLE_NAMES.events} WHERE ${cursor ? `created_at <= '${formatClickhouseDate(cursor)}' AND` : ''} project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY toDate(created_at) DESC, created_at DESC LIMIT 50;`,
         {
           profile: true,
           meta: true,
         },
       );
+
+      const lastItem = items[items.length - 1];
+
+      return {
+        items,
+        meta: {
+          next: lastItem ? lastItem.createdAt.toISOString() : null,
+        },
+      };
     }),
 
   bots: publicProcedure

packages/trpc/src/routers/overview.ts

@@ -0,0 +1,156 @@
+import {
+  overviewService,
+  zGetMetricsInput,
+  zGetTopGenericInput,
+  zGetTopPagesInput,
+} from '@openpanel/db';
+import { type IChartRange, zRange } from '@openpanel/validation';
+import { z } from 'zod';
+import { cacheMiddleware, createTRPCRouter, publicProcedure } from '../trpc';
+import {
+  getChartPrevStartEndDate,
+  getChartStartEndDate,
+} from './chart.helpers';
+
+const cacher = cacheMiddleware((input) => {
+  const range = input.range as IChartRange;
+  switch (range) {
+    case '30min':
+    case 'today':
+    case 'lastHour':
+      return 1;
+    default:
+      return 1;
+  }
+});
+
+function getCurrentAndPrevious<
+  T extends {
+    startDate?: string | null;
+    endDate?: string | null;
+    range: IChartRange;
+  },
+>(input: T, fetchPrevious = false) {
+  const current = getChartStartEndDate(input);
+  const previous = getChartPrevStartEndDate(current);
+
+  return async <R>(
+    fn: (input: T & { startDate: string; endDate: string }) => Promise<R>,
+  ): Promise<{
+    current: R;
+    previous: R | null;
+  }> => {
+    const res = await Promise.all([
+      fn({
+        ...input,
+        startDate: current.startDate,
+        endDate: current.endDate,
+      }),
+      fetchPrevious
+        ? fn({
+            ...input,
+            startDate: previous.startDate,
+            endDate: previous.endDate,
+          })
+        : Promise.resolve(null),
+    ]);
+
+    return {
+      current: res[0],
+      previous: res[1],
+    };
+  };
+}
+
+export const overviewRouter = createTRPCRouter({
+  stats: publicProcedure
+    .input(
+      zGetMetricsInput.omit({ startDate: true, endDate: true }).extend({
+        startDate: z.string().nullish(),
+        endDate: z.string().nullish(),
+        range: zRange,
+      }),
+    )
+    .use(cacher)
+    .query(async ({ ctx, input }) => {
+      const { current, previous } = await getCurrentAndPrevious(
+        input,
+        true,
+      )(overviewService.getMetrics.bind(overviewService));
+
+      return {
+        metrics: {
+          ...current.metrics,
+          prev_bounce_rate: previous?.metrics.bounce_rate || null,
+          prev_unique_visitors: previous?.metrics.unique_visitors || null,
+          prev_total_screen_views: previous?.metrics.total_screen_views || null,
+          prev_avg_session_duration:
+            previous?.metrics.avg_session_duration || null,
+          prev_views_per_session: previous?.metrics.views_per_session || null,
+          prev_total_sessions: previous?.metrics.total_sessions || null,
+        },
+        series: current.series.map((item) => {
+          const prev = previous?.series.find((p) => p.date === item.date);
+          return {
+            ...item,
+            prev_bounce_rate: prev?.bounce_rate,
+            prev_unique_visitors: prev?.unique_visitors,
+            prev_total_screen_views: prev?.total_screen_views,
+            prev_avg_session_duration: prev?.avg_session_duration,
+            prev_views_per_session: prev?.views_per_session,
+            prev_total_sessions: prev?.total_sessions,
+          };
+        }),
+      };
+    }),
+
+  topPages: publicProcedure
+    .input(
+      zGetTopPagesInput.omit({ startDate: true, endDate: true }).extend({
+        startDate: z.string().nullish(),
+        endDate: z.string().nullish(),
+        range: zRange,
+        mode: z.enum(['page', 'entry', 'exit', 'bot']),
+      }),
+    )
+    .use(cacher)
+    .query(async ({ input }) => {
+      const { current } = await getCurrentAndPrevious(
+        input,
+        false,
+      )(async (input) => {
+        if (input.mode === 'page') {
+          return overviewService.getTopPages(input);
+        }
+
+        if (input.mode === 'bot') {
+          return Promise.resolve([]);
+        }
+
+        return overviewService.getTopEntryExit({
+          ...input,
+          mode: input.mode,
+        });
+      });
+
+      return current;
+    }),
+
+  topGeneric: publicProcedure
+    .input(
+      zGetTopGenericInput.omit({ startDate: true, endDate: true }).extend({
+        startDate: z.string().nullish(),
+        endDate: z.string().nullish(),
+        range: zRange,
+      }),
+    )
+    .use(cacher)
+    .query(async ({ input }) => {
+      const { current } = await getCurrentAndPrevious(
+        input,
+        false,
+      )(overviewService.getTopGeneric.bind(overviewService));
+
+      return current;
+    }),
+});

packages/trpc/src/trpc.ts

@@ -140,3 +140,39 @@ export const protectedProcedure = t.procedure
   .use(enforceUserIsAuthed)
   .use(enforceAccess)
   .use(loggerMiddleware);
+
+const middlewareMarker = 'middlewareMarker' as 'middlewareMarker' & {
+  __brand: 'middlewareMarker';
+};
+
+export const cacheMiddleware = (cbOrTtl: number | ((input: any) => number)) =>
+  t.middleware(async ({ ctx, next, path, type, rawInput, input }) => {
+    if (type !== 'query') {
+      return next();
+    }
+    let key = `trpc:${path}:`;
+    if (rawInput) {
+      key += JSON.stringify(rawInput).replace(/\"/g, "'");
+    }
+    const cache = await getRedisCache().getJson(key);
+    if (cache) {
+      return {
+        ok: true,
+        data: cache,
+        ctx,
+        marker: middlewareMarker,
+      };
+    }
+    const result = await next();
+
+    // @ts-expect-error
+    if (result.data) {
+      getRedisCache().setJson(
+        key,
+        typeof cbOrTtl === 'function' ? cbOrTtl(input) : cbOrTtl,
+        // @ts-expect-error
+        result.data,
+      );
+    }
+    return result;
+  });