fix: optimize event buffer (#278)

* fix: how we fetch profiles in the buffer * perf: optimize event buffer * remove unused file * fix * wip * wip: try groupmq 2 * try simplified event buffer with duration calculation on the fly instead
2026-03-16 13:29:40 +01:00
parent 4736f8509d
commit 4483e464d1
46 changed files with 887 additions and 1841 deletions
--- a/packages/db/src/buffers/event-buffer.test.ts
+++ b/packages/db/src/buffers/event-buffer.test.ts
@@ -2,42 +2,8 @@ import { getRedisCache } from '@openpanel/redis';
 import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { ch } from '../clickhouse/client';

-// Mock transformEvent to avoid circular dependency with buffers -> services -> buffers
-vi.mock('../services/event.service', () => ({
-  transformEvent: (event: any) => ({
-    id: event.id ?? 'id',
-    name: event.name,
-    deviceId: event.device_id,
-    profileId: event.profile_id,
-    projectId: event.project_id,
-    sessionId: event.session_id,
-    properties: event.properties ?? {},
-    createdAt: new Date(event.created_at ?? Date.now()),
-    country: event.country,
-    city: event.city,
-    region: event.region,
-    longitude: event.longitude,
-    latitude: event.latitude,
-    os: event.os,
-    osVersion: event.os_version,
-    browser: event.browser,
-    browserVersion: event.browser_version,
-    device: event.device,
-    brand: event.brand,
-    model: event.model,
-    duration: event.duration ?? 0,
-    path: event.path ?? '',
-    origin: event.origin ?? '',
-    referrer: event.referrer,
-    referrerName: event.referrer_name,
-    referrerType: event.referrer_type,
-    meta: event.meta,
-    importedAt: undefined,
-    sdkName: event.sdk_name,
-    sdkVersion: event.sdk_version,
-    profile: event.profile,
-  }),
-}));
+// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
+vi.mock('../services/event.service', () => ({}));

 import { EventBuffer } from './event-buffer';

@@ -68,18 +34,16 @@ describe('EventBuffer', () => {
      created_at: new Date().toISOString(),
    } as any;

-    // Get initial count
    const initialCount = await eventBuffer.getBufferSize();

-    // Add event
-    await eventBuffer.add(event);
+    eventBuffer.add(event);
+    await eventBuffer.flush();

-    // Buffer counter should increase by 1
    const newCount = await eventBuffer.getBufferSize();
    expect(newCount).toBe(initialCount + 1);
  });

-  it('adds multiple screen_views - moves previous to buffer with duration', async () => {
+  it('adds screen_view directly to buffer queue', async () => {
    const t0 = Date.now();
    const sessionId = 'session_1';

@@ -99,60 +63,23 @@ describe('EventBuffer', () => {
      created_at: new Date(t0 + 1000).toISOString(),
    } as any;

-    const view3 = {
-      project_id: 'p1',
-      profile_id: 'u1',
-      session_id: sessionId,
-      name: 'screen_view',
-      created_at: new Date(t0 + 3000).toISOString(),
-    } as any;
-
-    // Add first screen_view
    const count1 = await eventBuffer.getBufferSize();
-    await eventBuffer.add(view1);

-    // Should be stored as "last" but NOT in queue yet
+    eventBuffer.add(view1);
+    await eventBuffer.flush();
+
+    // screen_view goes directly to buffer
    const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1); // No change in buffer
+    expect(count2).toBe(count1 + 1);

-    // Last screen_view should be retrievable
-    const last1 = await eventBuffer.getLastScreenView({
-      projectId: 'p1',
-      sessionId: sessionId,
-    });
-    expect(last1).not.toBeNull();
-    expect(last1!.createdAt.toISOString()).toBe(view1.created_at);
+    eventBuffer.add(view2);
+    await eventBuffer.flush();

-    // Add second screen_view
-    await eventBuffer.add(view2);
-
-    // Now view1 should be in buffer
    const count3 = await eventBuffer.getBufferSize();
-    expect(count3).toBe(count1 + 1);
-
-    // view2 should now be the "last"
-    const last2 = await eventBuffer.getLastScreenView({
-      projectId: 'p1',
-      sessionId: sessionId,
-    });
-    expect(last2!.createdAt.toISOString()).toBe(view2.created_at);
-
-    // Add third screen_view
-    await eventBuffer.add(view3);
-
-    // Now view2 should also be in buffer
-    const count4 = await eventBuffer.getBufferSize();
-    expect(count4).toBe(count1 + 2);
-
-    // view3 should now be the "last"
-    const last3 = await eventBuffer.getLastScreenView({
-      projectId: 'p1',
-      sessionId: sessionId,
-    });
-    expect(last3!.createdAt.toISOString()).toBe(view3.created_at);
+    expect(count3).toBe(count1 + 2);
  });

-  it('adds session_end - moves last screen_view and session_end to buffer', async () => {
+  it('adds session_end directly to buffer queue', async () => {
    const t0 = Date.now();
    const sessionId = 'session_2';

@@ -172,148 +99,44 @@ describe('EventBuffer', () => {
      created_at: new Date(t0 + 5000).toISOString(),
    } as any;

-    // Add screen_view
    const count1 = await eventBuffer.getBufferSize();
-    await eventBuffer.add(view);

-    // Should be stored as "last", not in buffer yet
+    eventBuffer.add(view);
+    eventBuffer.add(sessionEnd);
+    await eventBuffer.flush();
+
    const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1);
-
-    // Add session_end
-    await eventBuffer.add(sessionEnd);
-
-    // Both should now be in buffer (+2)
-    const count3 = await eventBuffer.getBufferSize();
-    expect(count3).toBe(count1 + 2);
-
-    // Last screen_view should be cleared
-    const last = await eventBuffer.getLastScreenView({
-      projectId: 'p2',
-      sessionId: sessionId,
-    });
-    expect(last).toBeNull();
-  });
-
-  it('session_end with no previous screen_view - only adds session_end to buffer', async () => {
-    const sessionId = 'session_3';
-
-    const sessionEnd = {
-      project_id: 'p3',
-      profile_id: 'u3',
-      session_id: sessionId,
-      name: 'session_end',
-      created_at: new Date().toISOString(),
-    } as any;
-
-    const count1 = await eventBuffer.getBufferSize();
-    await eventBuffer.add(sessionEnd);
-
-    // Only session_end should be in buffer (+1)
-    const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1 + 1);
-  });
-
-  it('gets last screen_view by profileId', async () => {
-    const view = {
-      project_id: 'p4',
-      profile_id: 'u4',
-      session_id: 'session_4',
-      name: 'screen_view',
-      path: '/home',
-      created_at: new Date().toISOString(),
-    } as any;
-
-    await eventBuffer.add(view);
-
-    // Query by profileId
-    const result = await eventBuffer.getLastScreenView({
-      projectId: 'p4',
-      profileId: 'u4',
-    });
-
-    expect(result).not.toBeNull();
-    expect(result!.name).toBe('screen_view');
-    expect(result!.path).toBe('/home');
-  });
-
-  it('gets last screen_view by sessionId', async () => {
-    const sessionId = 'session_5';
-    const view = {
-      project_id: 'p5',
-      profile_id: 'u5',
-      session_id: sessionId,
-      name: 'screen_view',
-      path: '/about',
-      created_at: new Date().toISOString(),
-    } as any;
-
-    await eventBuffer.add(view);
-
-    // Query by sessionId
-    const result = await eventBuffer.getLastScreenView({
-      projectId: 'p5',
-      sessionId: sessionId,
-    });
-
-    expect(result).not.toBeNull();
-    expect(result!.name).toBe('screen_view');
-    expect(result!.path).toBe('/about');
-  });
-
-  it('returns null for non-existent last screen_view', async () => {
-    const result = await eventBuffer.getLastScreenView({
-      projectId: 'p_nonexistent',
-      profileId: 'u_nonexistent',
-    });
-
-    expect(result).toBeNull();
+    expect(count2).toBe(count1 + 2);
  });

  it('gets buffer count correctly', async () => {
-    // Initially 0
    expect(await eventBuffer.getBufferSize()).toBe(0);

-    // Add regular event
-    await eventBuffer.add({
+    eventBuffer.add({
      project_id: 'p6',
      name: 'event1',
      created_at: new Date().toISOString(),
    } as any);
-
+    await eventBuffer.flush();
    expect(await eventBuffer.getBufferSize()).toBe(1);

-    // Add another regular event
-    await eventBuffer.add({
+    eventBuffer.add({
      project_id: 'p6',
      name: 'event2',
      created_at: new Date().toISOString(),
    } as any);
-
+    await eventBuffer.flush();
    expect(await eventBuffer.getBufferSize()).toBe(2);

-    // Add screen_view (not counted until flushed)
-    await eventBuffer.add({
+    // screen_view also goes directly to buffer
+    eventBuffer.add({
      project_id: 'p6',
      profile_id: 'u6',
      session_id: 'session_6',
      name: 'screen_view',
      created_at: new Date().toISOString(),
    } as any);
-
-    // Still 2 (screen_view is pending)
-    expect(await eventBuffer.getBufferSize()).toBe(2);
-
-    // Add another screen_view (first one gets flushed)
-    await eventBuffer.add({
-      project_id: 'p6',
-      profile_id: 'u6',
-      session_id: 'session_6',
-      name: 'screen_view',
-      created_at: new Date(Date.now() + 1000).toISOString(),
-    } as any);
-
-    // Now 3 (2 regular + 1 flushed screen_view)
+    await eventBuffer.flush();
    expect(await eventBuffer.getBufferSize()).toBe(3);
  });

@@ -330,8 +153,9 @@ describe('EventBuffer', () => {
      created_at: new Date(Date.now() + 1000).toISOString(),
    } as any;

-    await eventBuffer.add(event1);
-    await eventBuffer.add(event2);
+    eventBuffer.add(event1);
+    eventBuffer.add(event2);
+    await eventBuffer.flush();

    expect(await eventBuffer.getBufferSize()).toBe(2);

@@ -341,14 +165,12 @@ describe('EventBuffer', () => {

    await eventBuffer.processBuffer();

-    // Should insert both events
    expect(insertSpy).toHaveBeenCalled();
    const callArgs = insertSpy.mock.calls[0]![0];
    expect(callArgs.format).toBe('JSONEachRow');
    expect(callArgs.table).toBe('events');
    expect(Array.isArray(callArgs.values)).toBe(true);

-    // Buffer should be empty after processing
    expect(await eventBuffer.getBufferSize()).toBe(0);

    insertSpy.mockRestore();
@@ -359,14 +181,14 @@ describe('EventBuffer', () => {
    process.env.EVENT_BUFFER_CHUNK_SIZE = '2';
    const eb = new EventBuffer();

-    // Add 4 events
    for (let i = 0; i < 4; i++) {
-      await eb.add({
+      eb.add({
        project_id: 'p8',
        name: `event${i}`,
        created_at: new Date(Date.now() + i).toISOString(),
      } as any);
    }
+    await eb.flush();

    const insertSpy = vi
      .spyOn(ch, 'insert')
@@ -374,14 +196,12 @@ describe('EventBuffer', () => {

    await eb.processBuffer();

-    // With chunk size 2 and 4 events, should be called twice
    expect(insertSpy).toHaveBeenCalledTimes(2);
    const call1Values = insertSpy.mock.calls[0]![0].values as any[];
    const call2Values = insertSpy.mock.calls[1]![0].values as any[];
    expect(call1Values.length).toBe(2);
    expect(call2Values.length).toBe(2);

-    // Restore
    if (prev === undefined) delete process.env.EVENT_BUFFER_CHUNK_SIZE;
    else process.env.EVENT_BUFFER_CHUNK_SIZE = prev;

@@ -396,129 +216,61 @@ describe('EventBuffer', () => {
      created_at: new Date().toISOString(),
    } as any;

-    await eventBuffer.add(event);
+    eventBuffer.add(event);
+    await eventBuffer.flush();

    const count = await eventBuffer.getActiveVisitorCount('p9');
    expect(count).toBeGreaterThanOrEqual(1);
  });

-  it('handles multiple sessions independently', async () => {
+  it('handles multiple sessions independently — all events go to buffer', async () => {
    const t0 = Date.now();
+    const count1 = await eventBuffer.getBufferSize();

-    // Session 1
-    const view1a = {
+    eventBuffer.add({
      project_id: 'p10',
      profile_id: 'u10',
      session_id: 'session_10a',
      name: 'screen_view',
      created_at: new Date(t0).toISOString(),
-    } as any;
-
-    const view1b = {
-      project_id: 'p10',
-      profile_id: 'u10',
-      session_id: 'session_10a',
-      name: 'screen_view',
-      created_at: new Date(t0 + 1000).toISOString(),
-    } as any;
-
-    // Session 2
-    const view2a = {
+    } as any);
+    eventBuffer.add({
      project_id: 'p10',
      profile_id: 'u11',
      session_id: 'session_10b',
      name: 'screen_view',
      created_at: new Date(t0).toISOString(),
-    } as any;
-
-    const view2b = {
+    } as any);
+    eventBuffer.add({
+      project_id: 'p10',
+      profile_id: 'u10',
+      session_id: 'session_10a',
+      name: 'screen_view',
+      created_at: new Date(t0 + 1000).toISOString(),
+    } as any);
+    eventBuffer.add({
      project_id: 'p10',
      profile_id: 'u11',
      session_id: 'session_10b',
      name: 'screen_view',
      created_at: new Date(t0 + 2000).toISOString(),
-    } as any;
+    } as any);
+    await eventBuffer.flush();

-    await eventBuffer.add(view1a);
-    await eventBuffer.add(view2a);
-    await eventBuffer.add(view1b); // Flushes view1a
-    await eventBuffer.add(view2b); // Flushes view2a
-
-    // Should have 2 events in buffer (one from each session)
-    expect(await eventBuffer.getBufferSize()).toBe(2);
-
-    // Each session should have its own "last" screen_view
-    const last1 = await eventBuffer.getLastScreenView({
-      projectId: 'p10',
-      sessionId: 'session_10a',
-    });
-    expect(last1!.createdAt.toISOString()).toBe(view1b.created_at);
-
-    const last2 = await eventBuffer.getLastScreenView({
-      projectId: 'p10',
-      sessionId: 'session_10b',
-    });
-    expect(last2!.createdAt.toISOString()).toBe(view2b.created_at);
+    // All 4 events are in buffer directly
+    expect(await eventBuffer.getBufferSize()).toBe(count1 + 4);
  });

-  it('screen_view without session_id goes directly to buffer', async () => {
-    const view = {
+  it('bulk adds events to buffer', async () => {
+    const events = Array.from({ length: 5 }, (_, i) => ({
      project_id: 'p11',
-      profile_id: 'u11',
-      name: 'screen_view',
-      created_at: new Date().toISOString(),
-    } as any;
+      name: `event${i}`,
+      created_at: new Date(Date.now() + i).toISOString(),
+    })) as any[];

-    const count1 = await eventBuffer.getBufferSize();
-    await eventBuffer.add(view);
+    eventBuffer.bulkAdd(events);
+    await eventBuffer.flush();

-    // Should go directly to buffer (no session_id)
-    const count2 = await eventBuffer.getBufferSize();
-    expect(count2).toBe(count1 + 1);
-  });
-
-  it('updates last screen_view when new one arrives from same profile but different session', async () => {
-    const t0 = Date.now();
-
-    const view1 = {
-      project_id: 'p12',
-      profile_id: 'u12',
-      session_id: 'session_12a',
-      name: 'screen_view',
-      path: '/page1',
-      created_at: new Date(t0).toISOString(),
-    } as any;
-
-    const view2 = {
-      project_id: 'p12',
-      profile_id: 'u12',
-      session_id: 'session_12b', // Different session!
-      name: 'screen_view',
-      path: '/page2',
-      created_at: new Date(t0 + 1000).toISOString(),
-    } as any;
-
-    await eventBuffer.add(view1);
-    await eventBuffer.add(view2);
-
-    // Both sessions should have their own "last"
-    const lastSession1 = await eventBuffer.getLastScreenView({
-      projectId: 'p12',
-      sessionId: 'session_12a',
-    });
-    expect(lastSession1!.path).toBe('/page1');
-
-    const lastSession2 = await eventBuffer.getLastScreenView({
-      projectId: 'p12',
-      sessionId: 'session_12b',
-    });
-    expect(lastSession2!.path).toBe('/page2');
-
-    // Profile should have the latest one
-    const lastProfile = await eventBuffer.getLastScreenView({
-      projectId: 'p12',
-      profileId: 'u12',
-    });
-    expect(lastProfile!.path).toBe('/page2');
+    expect(await eventBuffer.getBufferSize()).toBe(5);
  });
 });
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/db/src/buffers/event-buffer.ts
@@ -2,33 +2,13 @@ import { getSafeJson } from '@openpanel/json';
 import {
  type Redis,
  getRedisCache,
-  getRedisPub,
  publishEvent,
 } from '@openpanel/redis';
 import { ch } from '../clickhouse/client';
-import {
-  type IClickhouseEvent,
-  type IServiceEvent,
-  transformEvent,
-} from '../services/event.service';
+import { type IClickhouseEvent } from '../services/event.service';
 import { BaseBuffer } from './base-buffer';

-/**
- * Simplified Event Buffer
- *
- * Rules:
- * 1. All events go into a single list buffer (event_buffer:queue)
- * 2. screen_view events are handled specially:
- *    - Store current screen_view as "last" for the session
- *    - When a new screen_view arrives, flush the previous one with calculated duration
- * 3. session_end events:
- *    - Retrieve the last screen_view (don't modify it)
- *    - Push both screen_view and session_end to buffer
- * 4. Flush: Simply process all events from the list buffer
- */
-
 export class EventBuffer extends BaseBuffer {
-  // Configurable limits
  private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE
    ? Number.parseInt(process.env.EVENT_BUFFER_BATCH_SIZE, 10)
    : 4000;
@@ -36,124 +16,26 @@ export class EventBuffer extends BaseBuffer {
    ? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)
    : 1000;

+  private microBatchIntervalMs = process.env.EVENT_BUFFER_MICRO_BATCH_MS
+    ? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_MS, 10)
+    : 10;
+  private microBatchMaxSize = process.env.EVENT_BUFFER_MICRO_BATCH_SIZE
+    ? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_SIZE, 10)
+    : 100;
+
+  private pendingEvents: IClickhouseEvent[] = [];
+  private flushTimer: ReturnType<typeof setTimeout> | null = null;
+  private isFlushing = false;
+  /** Tracks consecutive flush failures for observability; reset on success. */
+  private flushRetryCount = 0;
+
  private activeVisitorsExpiration = 60 * 5; // 5 minutes
-
-  // LIST - Stores all events ready to be flushed
+  /** How often (ms) we refresh the heartbeat key + zadd per visitor. */
+  private heartbeatRefreshMs = 60_000; // 1 minute
+  private lastHeartbeat = new Map<string, number>();
  private queueKey = 'event_buffer:queue';
-
-  // STRING - Tracks total buffer size incrementally
  protected bufferCounterKey = 'event_buffer:total_count';

-  // Script SHAs for loaded Lua scripts
-  private scriptShas: {
-    addScreenView?: string;
-    addSessionEnd?: string;
-  } = {};
-
-  // Hash key for storing last screen_view per session
-  private getLastScreenViewKeyBySession(sessionId: string) {
-    return `event_buffer:last_screen_view:session:${sessionId}`;
-  }
-
-  // Hash key for storing last screen_view per profile
-  private getLastScreenViewKeyByProfile(projectId: string, profileId: string) {
-    return `event_buffer:last_screen_view:profile:${projectId}:${profileId}`;
-  }
-
-  /**
-   * Lua script for handling screen_view addition - RACE-CONDITION SAFE without GroupMQ
-   *
-   * Strategy: Use Redis GETDEL (atomic get-and-delete) to ensure only ONE thread
-   * can process the "last" screen_view at a time.
-   *
-   * KEYS[1] = last screen_view key (by session) - stores both event and timestamp as JSON
-   * KEYS[2] = last screen_view key (by profile, may be empty)
-   * KEYS[3] = queue key
-   * KEYS[4] = buffer counter key
-   * ARGV[1] = new event with timestamp as JSON: {"event": {...}, "ts": 123456}
-   * ARGV[2] = TTL for last screen_view (1 hour)
-   */
-  private readonly addScreenViewScript = `
-local sessionKey = KEYS[1]
-local profileKey = KEYS[2]
-local queueKey = KEYS[3]
-local counterKey = KEYS[4]
-local newEventData = ARGV[1]
-local ttl = tonumber(ARGV[2])
-
-- GETDEL is atomic: get previous and delete in one operation
-- This ensures only ONE thread gets the previous event
-local previousEventData = redis.call("GETDEL", sessionKey)
-
-- Store new screen_view as last for session
-redis.call("SET", sessionKey, newEventData, "EX", ttl)
-
-- Store new screen_view as last for profile (if key provided)
-if profileKey and profileKey ~= "" then
-  redis.call("SET", profileKey, newEventData, "EX", ttl)
-end
-
-- If there was a previous screen_view, add it to queue with calculated duration
-if previousEventData then
-  local prev = cjson.decode(previousEventData)
-  local curr = cjson.decode(newEventData)
-  
-  -- Calculate duration (ensure non-negative to handle clock skew)
-  if prev.ts and curr.ts then
-    prev.event.duration = math.max(0, curr.ts - prev.ts)
-  end
-  
-  redis.call("RPUSH", queueKey, cjson.encode(prev.event))
-  redis.call("INCR", counterKey)
-  return 1
-end
-
-return 0
-`;
-
-  /**
-   * Lua script for handling session_end - RACE-CONDITION SAFE
-   *
-   * Uses GETDEL to atomically retrieve and delete the last screen_view
-   *
-   * KEYS[1] = last screen_view key (by session)
-   * KEYS[2] = last screen_view key (by profile, may be empty)
-   * KEYS[3] = queue key
-   * KEYS[4] = buffer counter key
-   * ARGV[1] = session_end event JSON
-   */
-  private readonly addSessionEndScript = `
-local sessionKey = KEYS[1]
-local profileKey = KEYS[2]
-local queueKey = KEYS[3]
-local counterKey = KEYS[4]
-local sessionEndJson = ARGV[1]
-
-- GETDEL is atomic: only ONE thread gets the last screen_view
-local previousEventData = redis.call("GETDEL", sessionKey)
-local added = 0
-
-- If there was a previous screen_view, add it to queue
-if previousEventData then
-  local prev = cjson.decode(previousEventData)
-  redis.call("RPUSH", queueKey, cjson.encode(prev.event))
-  redis.call("INCR", counterKey)
-  added = added + 1
-end
-
-- Add session_end to queue
-redis.call("RPUSH", queueKey, sessionEndJson)
-redis.call("INCR", counterKey)
-added = added + 1
-
-- Delete profile key
-if profileKey and profileKey ~= "" then
-  redis.call("DEL", profileKey)
-end
-
-return added
-`;
-
  constructor() {
    super({
      name: 'event',
@@ -161,170 +43,97 @@ return added
        await this.processBuffer();
      },
    });
-    // Load Lua scripts into Redis on startup
-    this.loadScripts();
-  }
-
-  /**
-   * Load Lua scripts into Redis and cache their SHAs.
-   * This avoids sending the entire script on every call.
-   */
-  private async loadScripts() {
-    try {
-      const redis = getRedisCache();
-      const [screenViewSha, sessionEndSha] = await Promise.all([
-        redis.script('LOAD', this.addScreenViewScript),
-        redis.script('LOAD', this.addSessionEndScript),
-      ]);
-
-      this.scriptShas.addScreenView = screenViewSha as string;
-      this.scriptShas.addSessionEnd = sessionEndSha as string;
-
-      this.logger.info('Loaded Lua scripts into Redis', {
-        addScreenView: this.scriptShas.addScreenView,
-        addSessionEnd: this.scriptShas.addSessionEnd,
-      });
-    } catch (error) {
-      this.logger.error('Failed to load Lua scripts', { error });
-    }
  }

  bulkAdd(events: IClickhouseEvent[]) {
-    const redis = getRedisCache();
-    const multi = redis.multi();
    for (const event of events) {
-      this.add(event, multi);
+      this.add(event);
    }
-    return multi.exec();
  }

-  /**
-   * Add an event into Redis buffer.
-   *
-   * Logic:
-   * - screen_view: Store as "last" for session, flush previous if exists
-   * - session_end: Flush last screen_view + session_end
-   * - Other events: Add directly to queue
-   */
-  async add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
+  add(event: IClickhouseEvent) {
+    this.pendingEvents.push(event);
+
+    if (this.pendingEvents.length >= this.microBatchMaxSize) {
+      this.flushLocalBuffer();
+      return;
+    }
+
+    if (!this.flushTimer) {
+      this.flushTimer = setTimeout(() => {
+        this.flushTimer = null;
+        this.flushLocalBuffer();
+      }, this.microBatchIntervalMs);
+    }
+  }
+
+  public async flush() {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    await this.flushLocalBuffer();
+  }
+
+  private async flushLocalBuffer() {
+    if (this.isFlushing || this.pendingEvents.length === 0) {
+      return;
+    }
+
+    this.isFlushing = true;
+
+    const eventsToFlush = this.pendingEvents;
+    this.pendingEvents = [];
+
    try {
      const redis = getRedisCache();
-      const eventJson = JSON.stringify(event);
-      const multi = _multi || redis.multi();
+      const multi = redis.multi();

-      if (event.session_id && event.name === 'screen_view') {
-        // Handle screen_view
-        const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
-        const profileKey = event.profile_id
-          ? this.getLastScreenViewKeyByProfile(
-              event.project_id,
-              event.profile_id,
-            )
-          : '';
-        const timestamp = new Date(event.created_at || Date.now()).getTime();
-
-        // Combine event and timestamp into single JSON for atomic operations
-        const eventWithTimestamp = JSON.stringify({
-          event: event,
-          ts: timestamp,
-        });
-
-        this.evalScript(
-          multi,
-          'addScreenView',
-          this.addScreenViewScript,
-          4,
-          sessionKey,
-          profileKey,
-          this.queueKey,
-          this.bufferCounterKey,
-          eventWithTimestamp,
-          '3600', // 1 hour TTL
-        );
-      } else if (event.session_id && event.name === 'session_end') {
-        // Handle session_end
-        const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
-        const profileKey = event.profile_id
-          ? this.getLastScreenViewKeyByProfile(
-              event.project_id,
-              event.profile_id,
-            )
-          : '';
-
-        this.evalScript(
-          multi,
-          'addSessionEnd',
-          this.addSessionEndScript,
-          4,
-          sessionKey,
-          profileKey,
-          this.queueKey,
-          this.bufferCounterKey,
-          eventJson,
-        );
-      } else {
-        // All other events go directly to queue
-        multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
+      for (const event of eventsToFlush) {
+        multi.rpush(this.queueKey, JSON.stringify(event));
+        if (event.profile_id) {
+          this.incrementActiveVisitorCount(
+            multi,
+            event.project_id,
+            event.profile_id,
+          );
+        }
      }
+      multi.incrby(this.bufferCounterKey, eventsToFlush.length);

-      if (event.profile_id) {
-        this.incrementActiveVisitorCount(
-          multi,
-          event.project_id,
-          event.profile_id,
-        );
-      }
+      await multi.exec();

-      if (!_multi) {
-        await multi.exec();
-      }
-
-      await publishEvent('events', 'received', transformEvent(event));
+      this.flushRetryCount = 0;
+      this.pruneHeartbeatMap();
    } catch (error) {
-      this.logger.error('Failed to add event to Redis buffer', { error });
+      // Re-queue failed events at the front to preserve order and avoid data loss
+      this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
+
+      this.flushRetryCount += 1;
+      this.logger.warn(
+        'Failed to flush local buffer to Redis; events re-queued',
+        {
+          error,
+          eventCount: eventsToFlush.length,
+          flushRetryCount: this.flushRetryCount,
+        },
+      );
+    } finally {
+      this.isFlushing = false;
+      // Events may have accumulated while we were flushing; schedule another flush if needed
+      if (this.pendingEvents.length > 0 && !this.flushTimer) {
+        this.flushTimer = setTimeout(() => {
+          this.flushTimer = null;
+          this.flushLocalBuffer();
+        }, this.microBatchIntervalMs);
+      }
    }
  }

-  /**
-   * Execute a Lua script using EVALSHA (cached) or fallback to EVAL.
-   * This avoids sending the entire script on every call.
-   */
-  private evalScript(
-    multi: ReturnType<Redis['multi']>,
-    scriptName: keyof typeof this.scriptShas,
-    scriptContent: string,
-    numKeys: number,
-    ...args: (string | number)[]
-  ) {
-    const sha = this.scriptShas[scriptName];
-
-    if (sha) {
-      // Use EVALSHA with cached SHA
-      multi.evalsha(sha, numKeys, ...args);
-    } else {
-      // Fallback to EVAL and try to reload script
-      multi.eval(scriptContent, numKeys, ...args);
-      this.logger.warn(`Script ${scriptName} not loaded, using EVAL fallback`);
-      // Attempt to reload scripts in background
-      this.loadScripts();
-    }
-  }
-
-  /**
-   * Process the Redis buffer - simplified version.
-   *
-   * Simply:
-   * 1. Fetch events from the queue (up to batchSize)
-   * 2. Parse and sort them
-   * 3. Insert into ClickHouse in chunks
-   * 4. Publish saved events
-   * 5. Clean up processed events from queue
-   */
  async processBuffer() {
    const redis = getRedisCache();

    try {
-      // Fetch events from queue
      const queueEvents = await redis.lrange(
        this.queueKey,
        0,
@@ -336,7 +145,6 @@ return added
        return;
      }

-      // Parse events
      const eventsToClickhouse: IClickhouseEvent[] = [];
      for (const eventStr of queueEvents) {
        const event = getSafeJson<IClickhouseEvent>(eventStr);
@@ -350,14 +158,12 @@ return added
        return;
      }

-      // Sort events by creation time
      eventsToClickhouse.sort(
        (a, b) =>
          new Date(a.created_at || 0).getTime() -
          new Date(b.created_at || 0).getTime(),
      );

-      // Insert events into ClickHouse in chunks
      this.logger.info('Inserting events into ClickHouse', {
        totalEvents: eventsToClickhouse.length,
        chunks: Math.ceil(eventsToClickhouse.length / this.chunkSize),
@@ -371,14 +177,17 @@ return added
        });
      }

-      // Publish "saved" events
-      const pubMulti = getRedisPub().multi();
+      const countByProject = new Map<string, number>();
      for (const event of eventsToClickhouse) {
-        await publishEvent('events', 'saved', transformEvent(event), pubMulti);
+        countByProject.set(
+          event.project_id,
+          (countByProject.get(event.project_id) ?? 0) + 1,
+        );
+      }
+      for (const [projectId, count] of countByProject) {
+        publishEvent('events', 'batch', { projectId, count });
      }
-      await pubMulti.exec();

-      // Clean up processed events from queue
      await redis
        .multi()
        .ltrim(this.queueKey, queueEvents.length, -1)
@@ -394,45 +203,6 @@ return added
    }
  }

-  /**
-   * Retrieve the latest screen_view event for a given session or profile
-   */
-  public async getLastScreenView(
-    params:
-      | {
-          sessionId: string;
-        }
-      | {
-          projectId: string;
-          profileId: string;
-        },
-  ): Promise<IServiceEvent | null> {
-    const redis = getRedisCache();
-
-    let lastScreenViewKey: string;
-    if ('sessionId' in params) {
-      lastScreenViewKey = this.getLastScreenViewKeyBySession(params.sessionId);
-    } else {
-      lastScreenViewKey = this.getLastScreenViewKeyByProfile(
-        params.projectId,
-        params.profileId,
-      );
-    }
-
-    const eventDataStr = await redis.get(lastScreenViewKey);
-
-    if (eventDataStr) {
-      const eventData = getSafeJson<{ event: IClickhouseEvent; ts: number }>(
-        eventDataStr,
-      );
-      if (eventData?.event) {
-        return transformEvent(eventData.event);
-      }
-    }
-
-    return null;
-  }
-
  public async getBufferSize() {
    return this.getBufferSizeWithCounter(async () => {
      const redis = getRedisCache();
@@ -440,16 +210,32 @@ return added
    });
  }

-  private async incrementActiveVisitorCount(
+  private pruneHeartbeatMap() {
+    const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
+    for (const [key, ts] of this.lastHeartbeat) {
+      if (ts < cutoff) {
+        this.lastHeartbeat.delete(key);
+      }
+    }
+  }
+
+  private incrementActiveVisitorCount(
    multi: ReturnType<Redis['multi']>,
    projectId: string,
    profileId: string,
  ) {
-    // Track active visitors and emit expiry events when inactive for TTL
+    const key = `${projectId}:${profileId}`;
    const now = Date.now();
+    const last = this.lastHeartbeat.get(key) ?? 0;
+
+    if (now - last < this.heartbeatRefreshMs) {
+      return;
+    }
+
+    this.lastHeartbeat.set(key, now);
    const zsetKey = `live:visitors:${projectId}`;
    const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
-    return multi
+    multi
      .zadd(zsetKey, now, profileId)
      .set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
  }
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/db/src/buffers/profile-buffer.ts
@@ -1,11 +1,11 @@
 import { deepMergeObjects } from '@openpanel/common';
 import { getSafeJson } from '@openpanel/json';
 import type { ILogger } from '@openpanel/logger';
-import { type Redis, getRedisCache } from '@openpanel/redis';
+import { getRedisCache, type Redis } from '@openpanel/redis';
 import shallowEqual from 'fast-deep-equal';
 import { omit } from 'ramda';
 import sqlstring from 'sqlstring';
-import { TABLE_NAMES, ch, chQuery } from '../clickhouse/client';
+import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
 import type { IClickhouseProfile } from '../services/profile.service';
 import { BaseBuffer } from './base-buffer';

@@ -89,7 +89,7 @@ export class ProfileBuffer extends BaseBuffer {
            'os_version',
            'browser_version',
          ],
-          profile.properties,
+          profile.properties
        );
      }

@@ -97,16 +97,16 @@ export class ProfileBuffer extends BaseBuffer {
        ? deepMergeObjects(existingProfile, omit(['created_at'], profile))
        : profile;

-      if (profile && existingProfile) {
-        if (
-          shallowEqual(
-            omit(['created_at'], existingProfile),
-            omit(['created_at'], mergedProfile),
-          )
-        ) {
-          this.logger.debug('Profile not changed, skipping');
-          return;
-        }
+      if (
+        profile &&
+        existingProfile &&
+        shallowEqual(
+          omit(['created_at'], existingProfile),
+          omit(['created_at'], mergedProfile)
+        )
+      ) {
+        this.logger.debug('Profile not changed, skipping');
+        return;
      }

      this.logger.debug('Merged profile will be inserted', {
@@ -151,11 +151,11 @@ export class ProfileBuffer extends BaseBuffer {

  private async fetchProfile(
    profile: IClickhouseProfile,
-    logger: ILogger,
+    logger: ILogger
  ): Promise<IClickhouseProfile | null> {
    const existingProfile = await this.fetchFromCache(
      profile.id,
-      profile.project_id,
+      profile.project_id
    );
    if (existingProfile) {
      logger.debug('Profile found in Redis');
@@ -167,7 +167,7 @@ export class ProfileBuffer extends BaseBuffer {

  public async fetchFromCache(
    profileId: string,
-    projectId: string,
+    projectId: string
  ): Promise<IClickhouseProfile | null> {
    const cacheKey = this.getProfileCacheKey({
      profileId,
@@ -182,7 +182,7 @@ export class ProfileBuffer extends BaseBuffer {

  private async fetchFromClickhouse(
    profile: IClickhouseProfile,
-    logger: ILogger,
+    logger: ILogger
  ): Promise<IClickhouseProfile | null> {
    logger.debug('Fetching profile from Clickhouse');
    const result = await chQuery<IClickhouseProfile>(
@@ -207,7 +207,7 @@ export class ProfileBuffer extends BaseBuffer {
        }
      GROUP BY id, project_id 
      ORDER BY created_at DESC 
-      LIMIT 1`,
+      LIMIT 1`
    );
    logger.debug('Clickhouse fetch result', {
      found: !!result[0],
@@ -221,7 +221,7 @@ export class ProfileBuffer extends BaseBuffer {
      const profiles = await this.redis.lrange(
        this.redisKey,
        0,
-        this.batchSize - 1,
+        this.batchSize - 1
      );

      if (profiles.length === 0) {
@@ -231,7 +231,7 @@ export class ProfileBuffer extends BaseBuffer {

      this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
      const parsedProfiles = profiles.map((p) =>
-        getSafeJson<IClickhouseProfile>(p),
+        getSafeJson<IClickhouseProfile>(p)
      );

      for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {
--- a/packages/db/src/services/clients.service.ts
+++ b/packages/db/src/services/clients.service.ts
@@ -1,4 +1,4 @@
-import { cacheable, cacheableLru } from '@openpanel/redis';
+import { cacheable } from '@openpanel/redis';
 import type { Client, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';

@@ -34,7 +34,4 @@ export async function getClientById(
  });
 }

-export const getClientByIdCached = cacheableLru(getClientById, {
-  maxSize: 1000,
-  ttl: 60 * 5,
-});
+export const getClientByIdCached = cacheable(getClientById, 60 * 5);
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -168,7 +168,6 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
    device: event.device,
    brand: event.brand,
    model: event.model,
-    duration: event.duration,
    path: event.path,
    origin: event.origin,
    referrer: event.referrer,
@@ -216,7 +215,7 @@ export interface IServiceEvent {
  device?: string | undefined;
  brand?: string | undefined;
  model?: string | undefined;
-  duration: number;
+  duration?: number;
  path: string;
  origin: string;
  referrer: string | undefined;
@@ -247,7 +246,7 @@ export interface IServiceEventMinimal {
  browser?: string | undefined;
  device?: string | undefined;
  brand?: string | undefined;
-  duration: number;
+  duration?: number;
  path: string;
  origin: string;
  referrer: string | undefined;
@@ -379,7 +378,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
    device: payload.device ?? '',
    brand: payload.brand ?? '',
    model: payload.model ?? '',
-    duration: payload.duration,
+    duration: payload.duration ?? 0,
    referrer: payload.referrer ?? '',
    referrer_name: payload.referrerName ?? '',
    referrer_type: payload.referrerType ?? '',
@@ -477,7 +476,7 @@ export async function getEventList(options: GetEventListOptions) {
    sb.where.cursor = `created_at < ${sqlstring.escape(formatClickhouseDate(cursor))}`;
  }

-  if (!cursor && !(startDate && endDate)) {
+  if (!(cursor || (startDate && endDate))) {
    sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;
  }

@@ -562,9 +561,6 @@ export async function getEventList(options: GetEventListOptions) {
  if (select.model) {
    sb.select.model = 'model';
  }
-  if (select.duration) {
-    sb.select.duration = 'duration';
-  }
  if (select.path) {
    sb.select.path = 'path';
  }
@@ -771,7 +767,6 @@ class EventService {
    where,
    select,
    limit,
-    orderBy,
    filters,
  }: {
    projectId: string;
@@ -811,7 +806,6 @@ class EventService {
        select.event.deviceId && 'e.device_id as device_id',
        select.event.name && 'e.name as name',
        select.event.path && 'e.path as path',
-        select.event.duration && 'e.duration as duration',
        select.event.country && 'e.country as country',
        select.event.city && 'e.city as city',
        select.event.os && 'e.os as os',
@@ -896,7 +890,6 @@ class EventService {
        select.event.deviceId && 'e.device_id as device_id',
        select.event.name && 'e.name as name',
        select.event.path && 'e.path as path',
-        select.event.duration && 'e.duration as duration',
        select.event.country && 'e.country as country',
        select.event.city && 'e.city as city',
        select.event.os && 'e.os as os',
@@ -1032,7 +1025,6 @@ class EventService {
          id: true,
          name: true,
          createdAt: true,
-          duration: true,
          country: true,
          city: true,
          os: true,
--- a/packages/db/src/services/notification.service.ts
+++ b/packages/db/src/services/notification.service.ts
@@ -90,7 +90,7 @@ export const getNotificationRulesByProjectId = cacheable(
      },
    });
  },
-  60 * 24
+  60 * 24,
 );

 function getIntegration(integrationId: string | null) {
--- a/packages/db/src/services/overview.service.ts
+++ b/packages/db/src/services/overview.service.ts
@@ -416,6 +416,30 @@ export class OverviewService {
    const where = this.getRawWhereClause('sessions', filters);
    const fillConfig = this.getFillConfig(interval, startDate, endDate);

+    // CTE: per-event screen_view durations via window function
+    const rawScreenViewDurationsQuery = clix(this.client, timezone)
+      .select([
+        `${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
+        `dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
+      ])
+      .from(TABLE_NAMES.events)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'screen_view')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ])
+      .rawWhere(this.getRawWhereClause('events', filters));
+
+    // CTE: avg duration per date bucket
+    const avgDurationByDateQuery = clix(this.client, timezone)
+      .select([
+        'date',
+        'round(avgIf(duration, duration > 0), 2) / 1000 AS avg_session_duration',
+      ])
+      .from('raw_screen_view_durations')
+      .groupBy(['date']);
+
    // Session aggregation with bounce rates
    const sessionAggQuery = clix(this.client, timezone)
      .select([
@@ -473,6 +497,8 @@ export class OverviewService {
          .where('date', '!=', rollupDate)
      )
      .with('overall_unique_visitors', overallUniqueVisitorsQuery)
+      .with('raw_screen_view_durations', rawScreenViewDurationsQuery)
+      .with('avg_duration_by_date', avgDurationByDateQuery)
      .select<{
        date: string;
        bounce_rate: number;
@@ -489,8 +515,7 @@ export class OverviewService {
        'dss.bounce_rate as bounce_rate',
        'uniq(e.profile_id) AS unique_visitors',
        'uniq(e.session_id) AS total_sessions',
-        'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
-        'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
+        'coalesce(dur.avg_session_duration, 0) AS avg_session_duration',
        'count(*) AS total_screen_views',
        'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
        '(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
@@ -502,6 +527,10 @@ export class OverviewService {
        'daily_session_stats AS dss',
        `${clix.toStartOf('e.created_at', interval as any)} = dss.date`
      )
+      .leftJoin(
+        'avg_duration_by_date AS dur',
+        `${clix.toStartOf('e.created_at', interval as any)} = dur.date`
+      )
      .where('e.project_id', '=', projectId)
      .where('e.name', '=', 'screen_view')
      .where('e.created_at', 'BETWEEN', [
@@ -509,7 +538,7 @@ export class OverviewService {
        clix.datetime(endDate, 'toDateTime'),
      ])
      .rawWhere(this.getRawWhereClause('events', filters))
-      .groupBy(['date', 'dss.bounce_rate'])
+      .groupBy(['date', 'dss.bounce_rate', 'dur.avg_session_duration'])
      .orderBy('date', 'ASC')
      .fill(fillConfig.from, fillConfig.to, fillConfig.step)
      .transform({
--- a/packages/db/src/services/pages.service.ts
+++ b/packages/db/src/services/pages.service.ts
@@ -52,6 +52,24 @@ export class PagesService {
      .where('created_at', '>=', clix.exp('now() - INTERVAL 30 DAY'))
      .groupBy(['origin', 'path']);

+    // CTE: compute screen_view durations via window function (leadInFrame gives next event's timestamp)
+    const screenViewDurationsCte = clix(this.client, timezone)
+      .select([
+        'project_id',
+        'session_id',
+        'path',
+        'origin',
+        `dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
+      ])
+      .from(TABLE_NAMES.events, false)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'screen_view')
+      .where('path', '!=', '')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ]);
+
    // Pre-filtered sessions subquery for better performance
    const sessionsSubquery = clix(this.client, timezone)
      .select(['id', 'project_id', 'is_bounce'])
@@ -66,6 +84,7 @@ export class PagesService {
    // Main query: aggregate events and calculate bounce rate from pre-filtered sessions
    const query = clix(this.client, timezone)
      .with('page_titles', titlesCte)
+      .with('screen_view_durations', screenViewDurationsCte)
      .select<ITopPage>([
        'e.origin as origin',
        'e.path as path',
@@ -74,25 +93,18 @@ export class PagesService {
        'count() as pageviews',
        'round(avg(e.duration) / 1000 / 60, 2) as avg_duration',
        `round(
-          (uniqIf(e.session_id, s.is_bounce = 1) * 100.0) / 
-          nullIf(uniq(e.session_id), 0), 
+          (uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /
+          nullIf(uniq(e.session_id), 0),
          2
        ) as bounce_rate`,
      ])
-      .from(`${TABLE_NAMES.events} e`, false)
+      .from('screen_view_durations e', false)
      .leftJoin(
        sessionsSubquery,
        'e.session_id = s.id AND e.project_id = s.project_id',
        's'
      )
      .leftJoin('page_titles pt', 'concat(e.origin, e.path) = pt.page_key')
-      .where('e.project_id', '=', projectId)
-      .where('e.name', '=', 'screen_view')
-      .where('e.path', '!=', '')
-      .where('e.created_at', 'BETWEEN', [
-        clix.datetime(startDate, 'toDateTime'),
-        clix.datetime(endDate, 'toDateTime'),
-      ])
      .when(!!search, (q) => {
        const term = `%${search}%`;
        q.whereGroup()
--- a/packages/db/src/services/project.service.ts
+++ b/packages/db/src/services/project.service.ts
@@ -1,6 +1,6 @@
 import { cacheable } from '@openpanel/redis';
 import sqlstring from 'sqlstring';
-import { TABLE_NAMES, chQuery } from '../clickhouse/client';
+import { chQuery, TABLE_NAMES } from '../clickhouse/client';
 import type { Prisma, Project } from '../prisma-client';
 import { db } from '../prisma-client';

@@ -25,6 +25,7 @@ export async function getProjectById(id: string) {
  return res;
 }

+/** L1 LRU (60s) + L2 Redis. clear() invalidates Redis + local LRU; other nodes may serve stale from LRU for up to 60s. */
 export const getProjectByIdCached = cacheable(getProjectById, 60 * 60 * 24);

 export async function getProjectWithClients(id: string) {
@@ -44,7 +45,7 @@ export async function getProjectWithClients(id: string) {
  return res;
 }

-export async function getProjectsByOrganizationId(organizationId: string) {
+export function getProjectsByOrganizationId(organizationId: string) {
  return db.project.findMany({
    where: {
      organizationId,
@@ -95,7 +96,7 @@ export async function getProjects({

  if (access.length > 0) {
    return projects.filter((project) =>
-      access.some((a) => a.projectId === project.id),
+      access.some((a) => a.projectId === project.id)
    );
  }

@@ -104,7 +105,7 @@ export async function getProjects({

 export const getProjectEventsCount = async (projectId: string) => {
  const res = await chQuery<{ count: number }>(
-    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
+    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
  );
  return res[0]?.count;
 };
--- a/packages/db/src/services/salt.service.ts
+++ b/packages/db/src/services/salt.service.ts
@@ -1,9 +1,9 @@
 import { generateSalt } from '@openpanel/common/server';

-import { cacheableLru } from '@openpanel/redis';
+import { cacheable } from '@openpanel/redis';
 import { db } from '../prisma-client';

-export const getSalts = cacheableLru(
+export const getSalts = cacheable(
  'op:salt',
  async () => {
    const [curr, prev] = await db.salt.findMany({
@@ -24,10 +24,7 @@ export const getSalts = cacheableLru(

    return salts;
  },
-  {
-    maxSize: 2,
-    ttl: 60 * 5,
-  },
+  60 * 5,
 );

 export async function createInitialSalts() {
--- a/packages/payments/scripts/assign-product-to-org.ts
+++ b/packages/payments/scripts/assign-product-to-org.ts
@@ -1,221 +0,0 @@
-import { db } from '@openpanel/db';
-import { Polar } from '@polar-sh/sdk';
-import inquirer from 'inquirer';
-import inquirerAutocomplete from 'inquirer-autocomplete-prompt';
-import { getSuccessUrl } from '..';
-
-// Register the autocomplete prompt
-inquirer.registerPrompt('autocomplete', inquirerAutocomplete);
-
-interface Answers {
-  isProduction: boolean;
-  polarApiKey: string;
-  productId: string;
-  organizationId: string;
-}
-
-async function promptForInput() {
-  // Get all organizations first
-  const organizations = await db.organization.findMany({
-    select: {
-      id: true,
-      name: true,
-    },
-  });
-
-  // Step 1: Collect Polar credentials first
-  const polarCredentials = await inquirer.prompt<{
-    isProduction: boolean;
-    polarApiKey: string;
-    polarOrganizationId: string;
-  }>([
-    {
-      type: 'list',
-      name: 'isProduction',
-      message: 'Is this for production?',
-      choices: [
-        { name: 'Yes', value: true },
-        { name: 'No', value: false },
-      ],
-      default: true,
-    },
-    {
-      type: 'string',
-      name: 'polarApiKey',
-      message: 'Enter your Polar API key:',
-      validate: (input: string) => {
-        if (!input) return 'API key is required';
-        return true;
-      },
-    },
-  ]);
-
-  // Step 2: Initialize Polar client and fetch products
-  const polar = new Polar({
-    accessToken: polarCredentials.polarApiKey,
-    server: polarCredentials.isProduction ? 'production' : 'sandbox',
-  });
-
-  console.log('Fetching products from Polar...');
-  const productsResponse = await polar.products.list({
-    limit: 100,
-    isArchived: false,
-    sorting: ['price_amount'],
-  });
-
-  const products = productsResponse.result.items;
-
-  if (products.length === 0) {
-    throw new Error('No products found in Polar');
-  }
-
-  // Step 3: Continue with product selection and organization selection
-  const restOfAnswers = await inquirer.prompt<{
-    productId: string;
-    organizationId: string;
-  }>([
-    {
-      type: 'autocomplete',
-      name: 'productId',
-      message: 'Select product:',
-      source: (answersSoFar: any, input = '') => {
-        return products
-          .filter(
-            (product) =>
-              product.name.toLowerCase().includes(input.toLowerCase()) ||
-              product.id.toLowerCase().includes(input.toLowerCase()),
-          )
-          .map((product) => {
-            const price = product.prices?.[0];
-            const priceStr =
-              price && 'priceAmount' in price && price.priceAmount
-                ? `$${(price.priceAmount / 100).toFixed(2)}/${price.recurringInterval || 'month'}`
-                : 'No price';
-            return {
-              name: `${product.name} (${priceStr})`,
-              value: product.id,
-            };
-          });
-      },
-    },
-    {
-      type: 'autocomplete',
-      name: 'organizationId',
-      message: 'Select organization:',
-      source: (answersSoFar: any, input = '') => {
-        return organizations
-          .filter(
-            (org) =>
-              org.name.toLowerCase().includes(input.toLowerCase()) ||
-              org.id.toLowerCase().includes(input.toLowerCase()),
-          )
-          .map((org) => ({
-            name: `${org.name} (${org.id})`,
-            value: org.id,
-          }));
-      },
-    },
-  ]);
-
-  return {
-    ...polarCredentials,
-    ...restOfAnswers,
-  };
-}
-
-async function main() {
-  console.log('Assigning existing product to organization...');
-  const input = await promptForInput();
-
-  const polar = new Polar({
-    accessToken: input.polarApiKey,
-    server: input.isProduction ? 'production' : 'sandbox',
-  });
-
-  const organization = await db.organization.findUniqueOrThrow({
-    where: {
-      id: input.organizationId,
-    },
-    select: {
-      id: true,
-      name: true,
-      createdBy: {
-        select: {
-          id: true,
-          email: true,
-          firstName: true,
-          lastName: true,
-        },
-      },
-      projects: {
-        select: {
-          id: true,
-        },
-      },
-    },
-  });
-
-  if (!organization.createdBy) {
-    throw new Error(
-      `Organization ${organization.name} does not have a creator. Cannot proceed.`,
-    );
-  }
-
-  const user = organization.createdBy;
-
-  // Fetch product details for review
-  const product = await polar.products.get({ id: input.productId });
-  const price = product.prices?.[0];
-  const priceStr =
-    price && 'priceAmount' in price && price.priceAmount
-      ? `$${(price.priceAmount / 100).toFixed(2)}/${price.recurringInterval || 'month'}`
-      : 'No price';
-
-  console.log('\nReview the following settings:');
-  console.table({
-    product: product.name,
-    price: priceStr,
-    organization: organization.name,
-    email: user.email,
-    name:
-      [user.firstName, user.lastName].filter(Boolean).join(' ') || 'No name',
-  });
-
-  const { confirmed } = await inquirer.prompt([
-    {
-      type: 'confirm',
-      name: 'confirmed',
-      message: 'Do you want to proceed?',
-      default: false,
-    },
-  ]);
-
-  if (!confirmed) {
-    console.log('Operation canceled');
-    return;
-  }
-
-  const checkoutLink = await polar.checkoutLinks.create({
-    paymentProcessor: 'stripe',
-    productId: input.productId,
-    allowDiscountCodes: false,
-    metadata: {
-      organizationId: organization.id,
-      userId: user.id,
-    },
-    successUrl: getSuccessUrl(
-      input.isProduction
-        ? 'https://dashboard.openpanel.dev'
-        : 'http://localhost:3000',
-      organization.id,
-    ),
-  });
-
-  console.log('\nCheckout link created:');
-  console.table(checkoutLink);
-  console.log('\nProduct assigned successfully!');
-}
-
-main()
-  .catch(console.error)
-  .finally(() => db.$disconnect());
--- a/packages/queue/src/queues.ts
+++ b/packages/queue/src/queues.ts
@@ -6,7 +6,7 @@ import type {
 } from '@openpanel/db';
 import { createLogger } from '@openpanel/logger';
 import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
-import { Queue, QueueEvents } from 'bullmq';
+import { Queue } from 'bullmq';
 import { Queue as GroupQueue } from 'groupmq';
 import type { ITrackPayload } from '../../validation';

@@ -66,6 +66,10 @@ export interface EventsQueuePayloadIncomingEvent {
    headers: Record<string, string | undefined>;
    deviceId: string;
    sessionId: string;
+    session?: Pick<
+      IServiceCreateEventPayload,
+      'referrer' | 'referrerName' | 'referrerType'
+    >;
  };
 }
 export interface EventsQueuePayloadCreateEvent {
@@ -206,9 +210,6 @@ export const sessionsQueue = new Queue<SessionsQueuePayload>(
    },
  }
 );
-export const sessionsQueueEvents = new QueueEvents(getQueueName('sessions'), {
-  connection: getRedisQueue(),
-});

 export const cronQueue = new Queue<CronQueuePayload>(getQueueName('cron'), {
  connection: getRedisQueue(),
--- a/packages/redis/cachable.ts
+++ b/packages/redis/cachable.ts
@@ -1,7 +1,7 @@
 import { LRUCache } from 'lru-cache';
 import { getRedisCache } from './redis';

-export const deleteCache = async (key: string) => {
+export const deleteCache = (key: string) => {
  return getRedisCache().del(key);
 };

@@ -15,7 +15,7 @@ export async function getCache<T>(
  key: string,
  expireInSec: number,
  fn: () => Promise<T>,
-  useLruCache?: boolean,
+  useLruCache?: boolean
 ): Promise<T> {
  // L1 Cache: Check global LRU cache first (in-memory, instant)
  if (useLruCache) {
@@ -28,15 +28,7 @@ export async function getCache<T>(
  // L2 Cache: Check Redis cache (shared across instances)
  const hit = await getRedisCache().get(key);
  if (hit) {
-    const parsed = JSON.parse(hit, (_, value) => {
-      if (
-        typeof value === 'string' &&
-        /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
-      ) {
-        return new Date(value);
-      }
-      return value;
-    });
+    const parsed = parseCache(hit);

    // Store in LRU cache for next time
    if (useLruCache) {
@@ -81,12 +73,24 @@ export function getGlobalLruCacheStats() {
 }

 function stringify(obj: unknown): string {
-  if (obj === null) return 'null';
-  if (obj === undefined) return 'undefined';
-  if (typeof obj === 'boolean') return obj ? 'true' : 'false';
-  if (typeof obj === 'number') return String(obj);
-  if (typeof obj === 'string') return obj;
-  if (typeof obj === 'function') return obj.toString();
+  if (obj === null) {
+    return 'null';
+  }
+  if (obj === undefined) {
+    return 'undefined';
+  }
+  if (typeof obj === 'boolean') {
+    return obj ? 'true' : 'false';
+  }
+  if (typeof obj === 'number') {
+    return String(obj);
+  }
+  if (typeof obj === 'string') {
+    return obj;
+  }
+  if (typeof obj === 'function') {
+    return obj.toString();
+  }

  if (Array.isArray(obj)) {
    return `[${obj.map(stringify).join(',')}]`;
@@ -128,17 +132,29 @@ function hasResult(result: unknown): boolean {
  return true;
 }

-export interface CacheableLruOptions {
-  /** TTL in seconds for LRU cache */
-  ttl: number;
-  /** Maximum number of entries in LRU cache */
-  maxSize?: number;
-}
+const DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/;
+const parseCache = (cached: string) => {
+  try {
+    return JSON.parse(cached, (_, value) => {
+      if (typeof value === 'string' && DATE_REGEX.test(value)) {
+        return new Date(value);
+      }
+      return value;
+    });
+  } catch (error) {
+    console.error('Failed to parse cache', error);
+    return null;
+  }
+};
+
+// L1 cache: short TTL to offload Redis; clear() invalidates Redis, other nodes may serve stale from LRU for up to this long
+const CACHEABLE_LRU_TTL_MS = 60 * 1000; // 60 seconds
+const CACHEABLE_LRU_MAX = 1000;

 // Overload 1: cacheable(fn, expireInSec)
 export function cacheable<T extends (...args: any) => any>(
  fn: T,
-  expireInSec: number,
+  expireInSec: number
 ): T & {
  getKey: (...args: Parameters<T>) => string;
  clear: (...args: Parameters<T>) => Promise<number>;
@@ -151,7 +167,7 @@ export function cacheable<T extends (...args: any) => any>(
 export function cacheable<T extends (...args: any) => any>(
  name: string,
  fn: T,
-  expireInSec: number,
+  expireInSec: number
 ): T & {
  getKey: (...args: Parameters<T>) => string;
  clear: (...args: Parameters<T>) => Promise<number>;
@@ -164,7 +180,7 @@ export function cacheable<T extends (...args: any) => any>(
 export function cacheable<T extends (...args: any) => any>(
  fnOrName: T | string,
  fnOrExpireInSec: number | T,
-  _expireInSec?: number,
+  _expireInSec?: number
 ) {
  const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
  const fn =
@@ -195,184 +211,67 @@ export function cacheable<T extends (...args: any) => any>(

  const cachePrefix = `cachable:${name}`;
  const getKey = (...args: Parameters<T>) =>
-    `${cachePrefix}:${stringify(args)}`;
+    `${cachePrefix}:${stringify(args)}`.replaceAll(/\s/g, '');

-  // Redis-only mode: asynchronous implementation
+  const lruCache = new LRUCache<string, any>({
+    max: CACHEABLE_LRU_MAX,
+    ttl: CACHEABLE_LRU_TTL_MS,
+  });
+
+  // L1 LRU (60s) + L2 Redis. clear() deletes Redis + local LRU; other nodes may serve stale from LRU for up to 60s.
  const cachedFn = async (
    ...args: Parameters<T>
  ): Promise<Awaited<ReturnType<T>>> => {
    const key = getKey(...args);

-    // Check Redis cache (shared across instances)
+    // L1: in-memory LRU first (offloads Redis on hot keys)
+    const lruHit = lruCache.get(key);
+    if (lruHit !== undefined && hasResult(lruHit)) {
+      return lruHit as Awaited<ReturnType<T>>;
+    }
+
+    // L2: Redis (shared across instances)
    const cached = await getRedisCache().get(key);
    if (cached) {
-      try {
-        const parsed = JSON.parse(cached, (_, value) => {
-          if (
-            typeof value === 'string' &&
-            /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*Z$/.test(value)
-          ) {
-            return new Date(value);
-          }
-          return value;
-        });
-        if (hasResult(parsed)) {
-          return parsed;
-        }
-      } catch (e) {
-        console.error('Failed to parse cache', e);
+      const parsed = parseCache(cached);
+      if (hasResult(parsed)) {
+        lruCache.set(key, parsed);
+        return parsed;
      }
    }

-    // Cache miss: Execute function
+    // Cache miss: execute function
    const result = await fn(...(args as any));

    if (hasResult(result)) {
-      // Don't await Redis write - fire and forget for better performance
+      lruCache.set(key, result);
      getRedisCache()
        .setex(key, expireInSec, JSON.stringify(result))
-        .catch(() => {});
+        .catch(() => {
+          // ignore error
+        });
    }

    return result;
  };

-  cachedFn.getKey = getKey;
-  cachedFn.clear = async (...args: Parameters<T>) => {
-    const key = getKey(...args);
-    return getRedisCache().del(key);
-  };
-  cachedFn.set =
-    (...args: Parameters<T>) =>
-    async (payload: Awaited<ReturnType<T>>) => {
-      const key = getKey(...args);
-      return getRedisCache()
-        .setex(key, expireInSec, JSON.stringify(payload))
-        .catch(() => {});
-    };
-
-  return cachedFn;
-}
-
-// Overload 1: cacheableLru(fn, options)
-export function cacheableLru<T extends (...args: any) => any>(
-  fn: T,
-  options: CacheableLruOptions,
-): T & {
-  getKey: (...args: Parameters<T>) => string;
-  clear: (...args: Parameters<T>) => boolean;
-  set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
-};
-
-// Overload 2: cacheableLru(name, fn, options)
-export function cacheableLru<T extends (...args: any) => any>(
-  name: string,
-  fn: T,
-  options: CacheableLruOptions,
-): T & {
-  getKey: (...args: Parameters<T>) => string;
-  clear: (...args: Parameters<T>) => boolean;
-  set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
-};
-
-// Implementation for cacheableLru (LRU-only - synchronous)
-export function cacheableLru<T extends (...args: any) => any>(
-  fnOrName: T | string,
-  fnOrOptions: T | CacheableLruOptions,
-  _options?: CacheableLruOptions,
-) {
-  const name = typeof fnOrName === 'string' ? fnOrName : fnOrName.name;
-  const fn =
-    typeof fnOrName === 'function'
-      ? fnOrName
-      : typeof fnOrOptions === 'function'
-        ? fnOrOptions
-        : null;
-
-  let options: CacheableLruOptions;
-
-  // Parse parameters based on function signature
-  if (typeof fnOrName === 'function') {
-    // Overload 1: cacheableLru(fn, options)
-    options =
-      typeof fnOrOptions === 'object' && fnOrOptions !== null
-        ? fnOrOptions
-        : ({} as CacheableLruOptions);
-  } else {
-    // Overload 2: cacheableLru(name, fn, options)
-    options =
-      typeof _options === 'object' && _options !== null
-        ? _options
-        : ({} as CacheableLruOptions);
-  }
-
-  if (typeof fn !== 'function') {
-    throw new Error('fn is not a function');
-  }
-
-  if (typeof options.ttl !== 'number') {
-    throw new Error('options.ttl is required and must be a number');
-  }
-
-  const cachePrefix = `cachable:${name}`;
-  const getKey = (...args: Parameters<T>) =>
-    `${cachePrefix}:${stringify(args)}`;
-
-  const maxSize = options.maxSize ?? 1000;
-  const ttl = options.ttl;
-
-  // Create function-specific LRU cache
-  const functionLruCache = new LRUCache<string, any>({
-    max: maxSize,
-    ttl: ttl * 1000, // Convert seconds to milliseconds for LRU
-  });
-
-  // LRU-only mode: synchronous implementation (or returns promise if fn is async)
-  const cachedFn = ((...args: Parameters<T>): ReturnType<T> => {
-    const key = getKey(...args);
-
-    // Check LRU cache
-    const lruHit = functionLruCache.get(key);
-    if (lruHit !== undefined && hasResult(lruHit)) {
-      return lruHit as ReturnType<T>;
-    }
-
-    // Cache miss: Execute function
-    const result = fn(...(args as any)) as ReturnType<T>;
-
-    // If result is a Promise, handle it asynchronously but cache the resolved value
-    if (result && typeof (result as any).then === 'function') {
-      return (result as Promise<any>).then((resolved: any) => {
-        if (hasResult(resolved)) {
-          functionLruCache.set(key, resolved);
-        }
-        return resolved;
-      }) as ReturnType<T>;
-    }
-
-    // Synchronous result: cache and return
-    if (hasResult(result)) {
-      functionLruCache.set(key, result);
-    }
-
-    return result;
-  }) as T & {
-    getKey: (...args: Parameters<T>) => string;
-    clear: (...args: Parameters<T>) => boolean;
-    set: (...args: Parameters<T>) => (payload: ReturnType<T>) => void;
-  };
-
  cachedFn.getKey = getKey;
  cachedFn.clear = (...args: Parameters<T>) => {
    const key = getKey(...args);
-    return functionLruCache.delete(key);
+    lruCache.delete(key);
+    return getRedisCache().del(key);
  };
  cachedFn.set =
    (...args: Parameters<T>) =>
-    (payload: ReturnType<T>) => {
+    (payload: Awaited<ReturnType<T>>) => {
      const key = getKey(...args);
      if (hasResult(payload)) {
-        functionLruCache.set(key, payload);
+        lruCache.set(key, payload);
+        return getRedisCache()
+          .setex(key, expireInSec, JSON.stringify(payload))
+          .catch(() => {
+            // ignore error
+          });
      }
    };

--- a/packages/redis/publisher.ts
+++ b/packages/redis/publisher.ts
@@ -10,8 +10,7 @@ export type IPublishChannels = {
    };
  };
  events: {
-    received: IServiceEvent;
-    saved: IServiceEvent;
+    batch: { projectId: string; count: number };
  };
  notification: {
    created: Prisma.NotificationUncheckedCreateInput;
--- a/packages/sdks/react-native/index.ts
+++ b/packages/sdks/react-native/index.ts
@@ -1,13 +1,13 @@
+import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
+import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
 import * as Application from 'expo-application';
 import Constants from 'expo-constants';
 import { AppState, Platform } from 'react-native';

-import type { OpenPanelOptions, TrackProperties } from '@openpanel/sdk';
-import { OpenPanel as OpenPanelBase } from '@openpanel/sdk';
-
 export * from '@openpanel/sdk';

 export class OpenPanel extends OpenPanelBase {
+  private lastPath = '';
  constructor(public options: OpenPanelOptions) {
    super({
      ...options,
@@ -37,7 +37,12 @@ export class OpenPanel extends OpenPanelBase {
    });
  }

-  public screenView(route: string, properties?: TrackProperties): void {
+  track(name: string, properties?: TrackProperties) {
+    return super.track(name, { ...properties, __path: this.lastPath });
+  }
+
+  screenView(route: string, properties?: TrackProperties): void {
+    this.lastPath = route;
    super.track('screen_view', {
      ...properties,
      __path: route,
--- a/packages/sdks/web/src/index.ts
+++ b/packages/sdks/web/src/index.ts
@@ -58,7 +58,7 @@ export type OpenPanelOptions = OpenPanelBaseOptions & {

 function toCamelCase(str: string) {
  return str.replace(/([-_][a-z])/gi, ($1) =>
-    $1.toUpperCase().replace('-', '').replace('_', ''),
+    $1.toUpperCase().replace('-', '').replace('_', '')
  );
 }

@@ -114,7 +114,9 @@ export class OpenPanel extends OpenPanelBase {
        const sampled = Math.random() < sampleRate;
        if (sampled) {
          this.loadReplayModule().then((mod) => {
-            if (!mod) return;
+            if (!mod) {
+              return;
+            }
            mod.startReplayRecorder(this.options.sessionReplay!, (chunk) => {
              // Replay chunks go through send() and are queued when disabled or waitForProfile
              // until ready() is called (base SDK also queues replay until sessionId is set).
@@ -153,7 +155,10 @@ export class OpenPanel extends OpenPanelBase {
      // dead-code-eliminated in the library build.
      if (typeof __OPENPANEL_REPLAY_URL__ !== 'undefined') {
        const scriptEl = _replayScriptRef;
-        const url = this.options.sessionReplay?.scriptUrl || scriptEl?.src?.replace('.js', '-replay.js') || 'https://openpanel.dev/op1-replay.js';
+        const url =
+          this.options.sessionReplay?.scriptUrl ||
+          scriptEl?.src?.replace('.js', '-replay.js') ||
+          'https://openpanel.dev/op1-replay.js';

        // Already loaded (e.g. user included the script manually)
        if ((window as any).__openpanel_replay) {
@@ -287,11 +292,15 @@ export class OpenPanel extends OpenPanelBase {
    });
  }

+  track(name: string, properties?: TrackProperties) {
+    return super.track(name, { ...properties, __path: this.lastPath });
+  }
+
  screenView(properties?: TrackProperties): void;
  screenView(path: string, properties?: TrackProperties): void;
  screenView(
    pathOrProperties?: string | TrackProperties,
-    propertiesOrUndefined?: TrackProperties,
+    propertiesOrUndefined?: TrackProperties
  ): void {
    if (this.isServer()) {
      return;
@@ -322,7 +331,7 @@ export class OpenPanel extends OpenPanelBase {

  async flushRevenue() {
    const promises = this.pendingRevenues.map((pending) =>
-      super.revenue(pending.amount, pending.properties),
+      super.revenue(pending.amount, pending.properties)
    );
    await Promise.all(promises);
    this.clearRevenue();
@@ -343,7 +352,7 @@ export class OpenPanel extends OpenPanelBase {
      try {
        sessionStorage.setItem(
          'openpanel-pending-revenues',
-          JSON.stringify(this.pendingRevenues),
+          JSON.stringify(this.pendingRevenues)
        );
      } catch {}
    }
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -1,11 +1,7 @@
-import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
-import sqlstring from 'sqlstring';
-import { z } from 'zod';
-
+import { round } from '@openpanel/common';
 import {
-  type IClickhouseProfile,
-  type IServiceProfile,
-  TABLE_NAMES,
+  AggregateChartEngine,
+  ChartEngine,
  ch,
  chQuery,
  clix,
@@ -21,8 +17,11 @@ import {
  getReportById,
  getSelectPropertyKey,
  getSettingsForProject,
+  type IClickhouseProfile,
+  type IServiceProfile,
  onlyReportEvents,
  sankeyService,
+  TABLE_NAMES,
  validateShareAccess,
 } from '@openpanel/db';
 import {
@@ -33,15 +32,15 @@ import {
  zReportInput,
  zTimeInterval,
 } from '@openpanel/validation';
-
-import { round } from '@openpanel/common';
-import { AggregateChartEngine, ChartEngine } from '@openpanel/db';
 import {
  differenceInDays,
  differenceInMonths,
  differenceInWeeks,
  formatISO,
 } from 'date-fns';
+import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
+import sqlstring from 'sqlstring';
+import { z } from 'zod';
 import { getProjectAccess } from '../access';
 import { TRPCAccessError } from '../errors';
 import {
@@ -83,7 +82,7 @@ const chartProcedure = publicProcedure.use(
          session: ctx.session?.userId
            ? { userId: ctx.session.userId }
            : undefined,
-        },
+        }
      );
      if (!shareValidation.isValid) {
        throw TRPCAccessError('You do not have access to this share');
@@ -119,7 +118,7 @@ const chartProcedure = publicProcedure.use(
        report: null,
      },
    });
-  },
+  }
 );

 export const chartRouter = createTRPCRouter({
@@ -128,7 +127,7 @@ export const chartRouter = createTRPCRouter({
    .input(
      z.object({
        projectId: z.string(),
-      }),
+      })
    )
    .query(async ({ input: { projectId } }) => {
      const { timezone } = await getSettingsForProject(projectId);
@@ -151,7 +150,7 @@ export const chartRouter = createTRPCRouter({
        TO toStartOfDay(now()) 
        STEP INTERVAL 1 day
        SETTINGS session_timezone = '${timezone}'
-      `,
+      `
      );

      const metricsPromise = clix(ch, timezone)
@@ -185,7 +184,7 @@ export const chartRouter = createTRPCRouter({
          ? Math.round(
              ((metrics.months_3 - metrics.months_3_prev) /
                metrics.months_3_prev) *
-                100,
+                100
            )
          : null;

@@ -209,12 +208,12 @@ export const chartRouter = createTRPCRouter({
    .input(
      z.object({
        projectId: z.string(),
-      }),
+      })
    )
    .query(async ({ input: { projectId } }) => {
      const [events, meta] = await Promise.all([
        chQuery<{ name: string; count: number }>(
-          `SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`,
+          `SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`
        ),
        getEventMetasCached(projectId),
      ]);
@@ -238,7 +237,7 @@ export const chartRouter = createTRPCRouter({
      z.object({
        event: z.string().optional(),
        projectId: z.string(),
-      }),
+      })
    )
    .query(async ({ input: { projectId, event } }) => {
      const profiles = await clix(ch, 'UTC')
@@ -252,8 +251,8 @@ export const chartRouter = createTRPCRouter({
      const profileProperties = [
        ...new Set(
          profiles.flatMap((p) =>
-            Object.keys(p.properties).map((k) => `profile.properties.${k}`),
-          ),
+            Object.keys(p.properties).map((k) => `profile.properties.${k}`)
+          )
        ),
      ];

@@ -283,7 +282,6 @@ export const chartRouter = createTRPCRouter({
      });

      const fixedProperties = [
-        'duration',
        'revenue',
        'has_profile',
        'path',
@@ -316,7 +314,7 @@ export const chartRouter = createTRPCRouter({

      return pipe(
        sort<string>((a, b) => a.length - b.length),
-        uniq,
+        uniq
      )(properties);
    }),

@@ -326,9 +324,9 @@ export const chartRouter = createTRPCRouter({
        event: z.string(),
        property: z.string(),
        projectId: z.string(),
-      }),
+      })
    )
-    .query(async ({ input: { event, property, projectId, ...input } }) => {
+    .query(async ({ input: { event, property, projectId } }) => {
      if (property === 'has_profile') {
        return {
          values: ['true', 'false'],
@@ -378,7 +376,7 @@ export const chartRouter = createTRPCRouter({
              .from(TABLE_NAMES.profiles)
              .where('project_id', '=', projectId),
            'profile.id = profile_id',
-            'profile',
+            'profile'
          );
        }

@@ -389,8 +387,8 @@ export const chartRouter = createTRPCRouter({
            (data: typeof events) => map(prop('values'), data),
            flatten,
            uniq,
-            sort((a, b) => a.length - b.length),
-          )(events),
+            sort((a, b) => a.length - b.length)
+          )(events)
        );
      }

@@ -406,8 +404,8 @@ export const chartRouter = createTRPCRouter({
        z.object({
          shareId: z.string().optional(),
          id: z.string().optional(),
-        }),
-      ),
+        })
+      )
    )
    .query(async ({ input, ctx }) => {
      const chartInput = ctx.report
@@ -448,8 +446,8 @@ export const chartRouter = createTRPCRouter({
        z.object({
          shareId: z.string().optional(),
          id: z.string().optional(),
-        }),
-      ),
+        })
+      )
    )
    .query(async ({ input, ctx }) => {
      const chartInput = ctx.report
@@ -536,12 +534,10 @@ export const chartRouter = createTRPCRouter({
        z.object({
          shareId: z.string().optional(),
          id: z.string().optional(),
-        }),
-      ),
+        })
+      )
    )
-    .query(async ({ input, ctx }) => {
-      console.log('input', input);
-
+    .query(({ input, ctx }) => {
      const chartInput = ctx.report
        ? {
            ...ctx.report,
@@ -562,10 +558,10 @@ export const chartRouter = createTRPCRouter({
        z.object({
          shareId: z.string().optional(),
          id: z.string().optional(),
-        }),
-      ),
+        })
+      )
    )
-    .query(async ({ input, ctx }) => {
+    .query(({ input, ctx }) => {
      const chartInput = ctx.report
        ? {
            ...ctx.report,
@@ -593,7 +589,7 @@ export const chartRouter = createTRPCRouter({
        range: zRange,
        shareId: z.string().optional(),
        id: z.string().optional(),
-      }),
+      })
    )
    .query(async ({ input, ctx }) => {
      const projectId = ctx.report?.projectId ?? input.projectId;
@@ -647,7 +643,7 @@ export const chartRouter = createTRPCRouter({
          startDate,
          endDate,
        },
-        timezone,
+        timezone
      );
      const diffInterval = {
        minute: () => differenceInDays(dates.endDate, dates.startDate),
@@ -677,14 +673,14 @@ export const chartRouter = createTRPCRouter({
      const usersSelect = range(0, diffInterval + 1)
        .map(
          (index) =>
-            `groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`,
+            `groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`
        )
        .join(',\n');

      const countsSelect = range(0, diffInterval + 1)
        .map(
          (index) =>
-            `length(interval_${index}_users) AS interval_${index}_user_count`,
+            `length(interval_${index}_users) AS interval_${index}_user_count`
        )
        .join(',\n');

@@ -769,12 +765,10 @@ export const chartRouter = createTRPCRouter({
        interval: zTimeInterval.default('day'),
        series: zChartSeries,
        breakdowns: z.record(z.string(), z.string()).optional(),
-      }),
+      })
    )
    .query(async ({ input }) => {
-      const { timezone } = await getSettingsForProject(input.projectId);
      const { projectId, date, series } = input;
-      const limit = 100;
      const serie = series[0];

      if (!serie) {
@@ -813,7 +807,7 @@ export const chartRouter = createTRPCRouter({
      if (profileFields.length > 0) {
        // Extract top-level field names and select only what's needed
        const fieldsToSelect = uniq(
-          profileFields.map((f) => f.split('.')[0]),
+          profileFields.map((f) => f.split('.')[0])
        ).join(', ');
        sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
      }
@@ -836,7 +830,7 @@ export const chartRouter = createTRPCRouter({
      // Fetch profile details in batches to avoid exceeding ClickHouse max_query_size
      const ids = profileIds.map((p) => p.profile_id).filter(Boolean);
      const BATCH_SIZE = 200;
-      const profiles = [];
+      const profiles: IServiceProfile[] = [];
      for (let i = 0; i < ids.length; i += BATCH_SIZE) {
        const batch = ids.slice(i, i + BATCH_SIZE);
        const batchProfiles = await getProfilesCached(batch, projectId);
@@ -859,13 +853,13 @@ export const chartRouter = createTRPCRouter({
          .optional()
          .default(false)
          .describe(
-            'If true, show users who dropped off at this step. If false, show users who completed at least this step.',
+            'If true, show users who dropped off at this step. If false, show users who completed at least this step.'
          ),
        funnelWindow: z.number().optional(),
        funnelGroup: z.string().optional(),
        breakdowns: z.array(z.object({ name: z.string() })).optional(),
        range: zRange,
-      }),
+      })
    )
    .query(async ({ input }) => {
      const { timezone } = await getSettingsForProject(input.projectId);
@@ -911,15 +905,15 @@ export const chartRouter = createTRPCRouter({

      // Check for profile filters and add profile join if needed
      const profileFilters = funnelService.getProfileFilters(
-        eventSeries as IChartEvent[],
+        eventSeries as IChartEvent[]
      );
      if (profileFilters.length > 0) {
        const fieldsToSelect = uniq(
-          profileFilters.map((f) => f.split('.')[0]),
+          profileFilters.map((f) => f.split('.')[0])
        ).join(', ');
        funnelCte.leftJoin(
          `(SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile`,
-          'profile.id = events.profile_id',
+          'profile.id = events.profile_id'
        );
      }

@@ -934,7 +928,7 @@ export const chartRouter = createTRPCRouter({
        // `max(level) AS level` alias (ILLEGAL_AGGREGATION error).
        query.with(
          'funnel',
-          'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id',
+          'SELECT profile_id, max(level) AS level FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id'
        );
      } else {
        // For session grouping: filter out level = 0 inside the CTE
@@ -969,7 +963,7 @@ export const chartRouter = createTRPCRouter({
      // when there are many profile IDs to pass in the IN(...) clause
      const ids = profileIdsResult.map((p) => p.profile_id).filter(Boolean);
      const BATCH_SIZE = 500;
-      const profiles = [];
+      const profiles: IServiceProfile[] = [];
      for (let i = 0; i < ids.length; i += BATCH_SIZE) {
        const batch = ids.slice(i, i + BATCH_SIZE);
        const batchProfiles = await getProfilesCached(batch, projectId);
@@ -986,7 +980,7 @@ function processCohortData(
    total_first_event_count: number;
    [key: string]: any;
  }>,
-  diffInterval: number,
+  diffInterval: number
 ) {
  if (data.length === 0) {
    return [];
@@ -995,13 +989,13 @@ function processCohortData(
  const processed = data.map((row) => {
    const sum = row.total_first_event_count;
    const values = range(0, diffInterval + 1).map(
-      (index) => (row[`interval_${index}_user_count`] || 0) as number,
+      (index) => (row[`interval_${index}_user_count`] || 0) as number
    );

    return {
      cohort_interval: row.cohort_interval,
      sum,
-      values: values,
+      values,
      percentages: values.map((value) => (sum > 0 ? round(value / sum, 2) : 0)),
    };
  });
@@ -1041,10 +1035,10 @@ function processCohortData(
    cohort_interval: 'Weighted Average',
    sum: round(averageData.totalSum / processed.length, 0),
    percentages: averageData.percentages.map(({ sum, weightedSum }) =>
-      sum > 0 ? round(weightedSum / sum, 2) : 0,
+      sum > 0 ? round(weightedSum / sum, 2) : 0
    ),
    values: averageData.values.map(({ sum, weightedSum }) =>
-      sum > 0 ? round(weightedSum / sum, 0) : 0,
+      sum > 0 ? round(weightedSum / sum, 0) : 0
    ),
  };

--- a/packages/trpc/src/routers/project.ts
+++ b/packages/trpc/src/routers/project.ts
@@ -96,9 +96,7 @@ export const projectRouter = createTRPCRouter({
      });
      await Promise.all([
        getProjectByIdCached.clear(input.id),
-        res.clients.map((client) => {
-          getClientByIdCached.clear(client.id);
-        }),
+        ...res.clients.map((client) => getClientByIdCached.clear(client.id)),
      ]);
      return res;
    }),
--- a/packages/trpc/src/routers/realtime.ts
+++ b/packages/trpc/src/routers/realtime.ts
@@ -1,18 +1,15 @@
-import { z } from 'zod';
-
 import {
-  type EventMeta,
-  TABLE_NAMES,
  ch,
  chQuery,
  clix,
-  db,
  formatClickhouseDate,
-  getEventList,
+  type IClickhouseEvent,
+  TABLE_NAMES,
+  transformEvent,
 } from '@openpanel/db';
-
 import { subMinutes } from 'date-fns';
 import sqlstring from 'sqlstring';
+import { z } from 'zod';
 import { createTRPCRouter, protectedProcedure } from '../trpc';

 export const realtimeRouter = createTRPCRouter({
@@ -25,7 +22,7 @@ export const realtimeRouter = createTRPCRouter({
        long: number;
        lat: number;
      }>(
-        `SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`,
+        `SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
      );

      return res;
@@ -33,25 +30,18 @@ export const realtimeRouter = createTRPCRouter({
  activeSessions: protectedProcedure
    .input(z.object({ projectId: z.string() }))
    .query(async ({ input }) => {
-      return getEventList({
-        projectId: input.projectId,
-        take: 30,
-        select: {
-          name: true,
-          path: true,
-          origin: true,
-          referrer: true,
-          referrerName: true,
-          referrerType: true,
-          country: true,
-          device: true,
-          os: true,
-          browser: true,
-          createdAt: true,
-          profile: true,
-          meta: true,
-        },
-      });
+      const rows = await chQuery<IClickhouseEvent>(
+        `SELECT
+          name, session_id, created_at, path, origin, referrer, referrer_name,
+          country, city, region, os, os_version, browser, browser_version,
+          device
+        FROM ${TABLE_NAMES.events}
+        WHERE project_id = ${sqlstring.escape(input.projectId)}
+          AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}'
+        ORDER BY created_at DESC
+        LIMIT 50`
+      );
+      return rows.map(transformEvent);
    }),
  paths: protectedProcedure
    .input(z.object({ projectId: z.string() }))
@@ -76,7 +66,7 @@ export const realtimeRouter = createTRPCRouter({
        .where(
          'created_at',
          '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
        )
        .groupBy(['path', 'origin'])
        .orderBy('count', 'DESC')
@@ -106,7 +96,7 @@ export const realtimeRouter = createTRPCRouter({
        .where(
          'created_at',
          '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
        )
        .groupBy(['referrer_name'])
        .orderBy('count', 'DESC')
@@ -137,7 +127,7 @@ export const realtimeRouter = createTRPCRouter({
        .where(
          'created_at',
          '>=',
-          formatClickhouseDate(subMinutes(new Date(), 30)),
+          formatClickhouseDate(subMinutes(new Date(), 30))
        )
        .groupBy(['country', 'city'])
        .orderBy('count', 'DESC')