fix: overall perf improvements

* fix: ignore private ips * fix: performance related fixes * fix: simply event buffer * fix: default to 1 events queue shard * add: cleanup scripts * fix: comments * fix comments * fix * fix: groupmq * wip * fix: sync cachable * remove cluster names and add it behind env flag (if someone want to scale) * fix * wip * better logger * remove reqid and user agent * fix lock * remove wait_for_async_insert
2025-11-15 22:13:59 +01:00
parent 38cc53890a
commit da59622dce
66 changed files with 5042 additions and 3860 deletions
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -13,11 +13,11 @@
  "dependencies": {
    "@ai-sdk/anthropic": "^1.2.10",
    "@ai-sdk/openai": "^1.3.12",
-    "@fastify/compress": "^8.0.1",
+    "@fastify/compress": "^8.1.0",
    "@fastify/cookie": "^11.0.2",
-    "@fastify/cors": "^11.0.0",
-    "@fastify/rate-limit": "^10.2.2",
-    "@fastify/websocket": "^11.0.2",
+    "@fastify/cors": "^11.1.0",
+    "@fastify/rate-limit": "^10.3.0",
+    "@fastify/websocket": "^11.2.0",
    "@node-rs/argon2": "^2.0.2",
    "@openpanel/auth": "workspace:^",
    "@openpanel/common": "workspace:*",
@@ -35,10 +35,10 @@
    "@trpc/server": "^11.6.0",
    "ai": "^4.2.10",
    "fast-json-stable-hash": "^1.0.3",
-    "fastify": "^5.2.1",
+    "fastify": "^5.6.1",
    "fastify-metrics": "^12.1.0",
    "fastify-raw-body": "^5.0.0",
-    "groupmq": "1.0.0-next.19",
+    "groupmq": "1.1.0-next.6",
    "jsonwebtoken": "^9.0.2",
    "ramda": "^0.29.1",
    "sharp": "^0.33.5",
--- a/apps/api/scripts/get-bots.ts
+++ b/apps/api/scripts/get-bots.ts
@@ -7,6 +7,23 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 import yaml from 'js-yaml';

+// Regex special characters that indicate we need actual regex
+const regexSpecialChars = /[|^$.*+?(){}\[\]\\]/;
+
+function transformBots(bots: any[]): any[] {
+  return bots.map((bot) => {
+    const { regex, ...rest } = bot;
+    const hasRegexChars = regexSpecialChars.test(regex);
+
+    if (hasRegexChars) {
+      // Keep as regex
+      return { regex, ...rest };
+    }
+    // Convert to includes
+    return { includes: regex, ...rest };
+  });
+}
+
 async function main() {
  // Get document, or throw exception on error
  try {
@@ -14,6 +31,9 @@ async function main() {
      'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
    ).then((res) => res.text());

+    const parsedData = yaml.load(data) as any[];
+    const transformedBots = transformBots(parsedData);
+
    fs.writeFileSync(
      path.resolve(__dirname, '../src/bots/bots.ts'),
      [
@@ -21,11 +41,20 @@ async function main() {
        '',
        '// The data is fetch from device-detector https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
        '',
-        `const bots = ${JSON.stringify(yaml.load(data))} as const;`,
+        `const bots = ${JSON.stringify(transformedBots, null, 2)} as const;`,
        'export default bots;',
+        '',
      ].join('\n'),
      'utf-8',
    );
+
+    console.log(
+      `✅ Generated bots.ts with ${transformedBots.length} bot entries`,
+    );
+    const regexCount = transformedBots.filter((b) => 'regex' in b).length;
+    const includesCount = transformedBots.filter((b) => 'includes' in b).length;
+    console.log(`   - ${includesCount} simple string matches (includes)`);
+    console.log(`   - ${regexCount} regex patterns`);
  } catch (e) {
    console.log(e);
  }
--- a/apps/api/scripts/test.ts
+++ b/apps/api/scripts/test.ts
@@ -40,8 +40,6 @@ async function main() {
        properties: {
          hash: 'test-hash',
          'query.utm_source': 'test',
-          __reqId: `req_${Math.floor(Math.random() * 1000)}`,
-          __user_agent: 'Mozilla/5.0 (Test)',
        },
        created_at: formatClickhouseDate(eventTime),
        country: 'US',
--- a/apps/api/src/bots/bots.ts
+++ b/apps/api/src/bots/bots.ts
--- a/apps/api/src/bots/index.ts
+++ b/apps/api/src/bots/index.ts
@@ -1,19 +1,47 @@
+import { cacheable, cacheableLru } from '@openpanel/redis';
 import bots from './bots';

-export function isBot(ua: string) {
-  const res = bots.find((bot) => {
-    if (new RegExp(bot.regex).test(ua)) {
-      return true;
-    }
-    return false;
-  });
-
-  if (!res) {
-    return null;
+// Pre-compile regex patterns at module load time
+const compiledBots = bots.map((bot) => {
+  if ('regex' in bot) {
+    return {
+      ...bot,
+      compiledRegex: new RegExp(bot.regex),
+    };
  }
+  return bot;
+});

-  return {
-    name: res.name,
-    type: 'category' in res ? res.category : 'Unknown',
-  };
-}
+const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
+const includesBots = compiledBots.filter((bot) => 'includes' in bot);
+
+export const isBot = cacheableLru(
+  'is-bot',
+  (ua: string) => {
+    // Check simple string patterns first (fast)
+    for (const bot of includesBots) {
+      if (ua.includes(bot.includes)) {
+        return {
+          name: bot.name,
+          type: 'category' in bot ? bot.category : 'Unknown',
+        };
+      }
+    }
+
+    // Check regex patterns (slower)
+    for (const bot of regexBots) {
+      if (bot.compiledRegex.test(ua)) {
+        return {
+          name: bot.name,
+          type: 'category' in bot ? bot.category : 'Unknown',
+        };
+      }
+    }
+
+    return null;
+  },
+  {
+    maxSize: 1000,
+    ttl: 60 * 5,
+  },
+);
--- a/apps/api/src/controllers/event.controller.ts
+++ b/apps/api/src/controllers/event.controller.ts
@@ -2,10 +2,9 @@ import type { FastifyReply, FastifyRequest } from 'fastify';

 import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
 import { getSalts } from '@openpanel/db';
-import { eventsGroupQueue } from '@openpanel/queue';
+import { getEventsGroupQueueShard } from '@openpanel/queue';
 import type { PostEventPayload } from '@openpanel/sdk';

-import { checkDuplicatedEvent } from '@/utils/deduplicate';
 import { generateId } from '@openpanel/common';
 import { getGeoLocation } from '@openpanel/geo';
 import { getStringHeaders, getTimestamp } from './track.controller';
@@ -44,28 +43,22 @@ export async function postEvent(
    ua,
  });

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-        timestamp,
-        previousDeviceId,
-        currentDeviceId,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  const uaInfo = parseUserAgent(ua, request.body?.properties);
  const groupId = uaInfo.isServer
    ? request.body?.profileId
      ? `${projectId}:${request.body?.profileId}`
      : `${projectId}:${generateId()}`
    : currentDeviceId;
-  await eventsGroupQueue.add({
+  const jobId = [
+    request.body.name,
+    timestamp,
+    projectId,
+    currentDeviceId,
+    groupId,
+  ]
+    .filter(Boolean)
+    .join('-');
+  await getEventsGroupQueueShard(groupId).add({
    orderMs: new Date(timestamp).getTime(),
    data: {
      projectId,
@@ -75,11 +68,13 @@ export async function postEvent(
        timestamp,
        isTimestampFromThePast,
      },
+      uaInfo,
      geo,
      currentDeviceId,
      previousDeviceId,
    },
    groupId,
+    jobId,
  });

  reply.status(202).send('ok');
--- a/apps/api/src/controllers/live.controller.ts
+++ b/apps/api/src/controllers/live.controller.ts
@@ -4,7 +4,7 @@ import superjson from 'superjson';
 import type { WebSocket } from '@fastify/websocket';
 import {
  eventBuffer,
-  getProfileByIdCached,
+  getProfileById,
  transformMinimalEvent,
 } from '@openpanel/db';
 import { setSuperJson } from '@openpanel/json';
@@ -92,10 +92,7 @@ export async function wsProjectEvents(
    type,
    async (event) => {
      if (event.projectId === params.projectId) {
-        const profile = await getProfileByIdCached(
-          event.profileId,
-          event.projectId,
-        );
+        const profile = await getProfileById(event.profileId, event.projectId);
        socket.send(
          superjson.stringify(
            access
--- a/apps/api/src/controllers/misc.controller.ts
+++ b/apps/api/src/controllers/misc.controller.ts
@@ -132,7 +132,7 @@ async function processImage(
 ): Promise<Buffer> {
  // If it's an ICO file, just return it as-is (no conversion needed)
  if (originalUrl && isIcoFile(originalUrl, contentType)) {
-    logger.info('Serving ICO file directly', {
+    logger.debug('Serving ICO file directly', {
      originalUrl,
      bufferSize: buffer.length,
    });
@@ -140,7 +140,7 @@ async function processImage(
  }

  if (originalUrl && isSvgFile(originalUrl, contentType)) {
-    logger.info('Serving SVG file directly', {
+    logger.debug('Serving SVG file directly', {
      originalUrl,
      bufferSize: buffer.length,
    });
@@ -149,7 +149,7 @@ async function processImage(

  // If buffer isnt to big just return it as well
  if (buffer.length < 5000) {
-    logger.info('Serving image directly without processing', {
+    logger.debug('Serving image directly without processing', {
      originalUrl,
      bufferSize: buffer.length,
    });
@@ -193,7 +193,7 @@ async function processOgImage(
 ): Promise<Buffer> {
  // If buffer is small enough, return it as-is
  if (buffer.length < 10000) {
-    logger.info('Serving OG image directly without processing', {
+    logger.debug('Serving OG image directly without processing', {
      originalUrl,
      bufferSize: buffer.length,
    });
--- a/apps/api/src/controllers/profile.controller.ts
+++ b/apps/api/src/controllers/profile.controller.ts
@@ -1,7 +1,6 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { assocPath, pathOr } from 'ramda';

-import { checkDuplicatedEvent, isDuplicatedEvent } from '@/utils/deduplicate';
 import { parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, upsertProfile } from '@openpanel/db';
 import { getGeoLocation } from '@openpanel/geo';
@@ -16,41 +15,39 @@ export async function updateProfile(
  }>,
  reply: FastifyReply,
 ) {
-  const { profileId, properties, ...rest } = request.body;
+  const payload = request.body;
  const projectId = request.client!.projectId;
  if (!projectId) {
    return reply.status(400).send('No projectId');
  }
  const ip = request.clientIp;
  const ua = request.headers['user-agent']!;
-  const uaInfo = parseUserAgent(ua, properties);
+  const uaInfo = parseUserAgent(ua, payload.properties);
  const geo = await getGeoLocation(ip);

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  await upsertProfile({
-    id: profileId,
+    ...payload,
+    id: payload.profileId,
    isExternal: true,
    projectId,
    properties: {
-      ...(properties ?? {}),
-      ...(ip ? geo : {}),
-      ...uaInfo,
+      ...(payload.properties ?? {}),
+      country: geo.country,
+      city: geo.city,
+      region: geo.region,
+      longitude: geo.longitude,
+      latitude: geo.latitude,
+      os: uaInfo.os,
+      os_version: uaInfo.osVersion,
+      browser: uaInfo.browser,
+      browser_version: uaInfo.browserVersion,
+      device: uaInfo.device,
+      brand: uaInfo.brand,
+      model: uaInfo.model,
    },
-    ...rest,
  });

-  reply.status(202).send(profileId);
+  reply.status(202).send(payload.profileId);
 }

 export async function incrementProfileProperty(
@@ -65,18 +62,6 @@ export async function incrementProfileProperty(
    return reply.status(400).send('No projectId');
  }

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  const profile = await getProfileById(profileId, projectId);
  if (!profile) {
    return reply.status(404).send('Not found');
@@ -119,18 +104,6 @@ export async function decrementProfileProperty(
    return reply.status(400).send('No projectId');
  }

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  const profile = await getProfileById(profileId, projectId);
  if (!profile) {
    return reply.status(404).send('Not found');
--- a/apps/api/src/controllers/track.controller.ts
+++ b/apps/api/src/controllers/track.controller.ts
@@ -1,12 +1,11 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
-import { path, assocPath, pathOr, pick } from 'ramda';
+import { assocPath, pathOr, pick } from 'ramda';

-import { checkDuplicatedEvent } from '@/utils/deduplicate';
 import { generateId } from '@openpanel/common';
 import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
 import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
-import { eventsGroupQueue } from '@openpanel/queue';
+import { getEventsGroupQueueShard } from '@openpanel/queue';
 import type {
  DecrementPayload,
  IdentifyPayload,
@@ -37,10 +36,10 @@ export function getStringHeaders(headers: FastifyRequest['headers']) {
 }

 function getIdentity(body: TrackHandlerPayload): IdentifyPayload | undefined {
-  const identity = path<IdentifyPayload>(
-    ['properties', '__identify'],
-    body.payload,
-  );
+  const identity =
+    'properties' in body.payload
+      ? (body.payload?.properties?.__identify as IdentifyPayload | undefined)
+      : undefined;

  return (
    identity ||
@@ -56,27 +55,28 @@ export function getTimestamp(
  timestamp: FastifyRequest['timestamp'],
  payload: TrackHandlerPayload['payload'],
 ) {
-  const safeTimestamp = new Date(timestamp || Date.now()).toISOString();
-  const userDefinedTimestamp = path<string>(
-    ['properties', '__timestamp'],
-    payload,
-  );
+  const safeTimestamp = timestamp || Date.now();
+  const userDefinedTimestamp =
+    'properties' in payload
+      ? (payload?.properties?.__timestamp as string | undefined)
+      : undefined;

  if (!userDefinedTimestamp) {
    return { timestamp: safeTimestamp, isTimestampFromThePast: false };
  }

  const clientTimestamp = new Date(userDefinedTimestamp);
+  const clientTimestampNumber = clientTimestamp.getTime();

  if (
-    Number.isNaN(clientTimestamp.getTime()) ||
-    clientTimestamp > new Date(safeTimestamp)
+    Number.isNaN(clientTimestampNumber) ||
+    clientTimestampNumber > safeTimestamp
  ) {
    return { timestamp: safeTimestamp, isTimestampFromThePast: false };
  }

  return {
-    timestamp: clientTimestamp.toISOString(),
+    timestamp: clientTimestampNumber,
    isTimestampFromThePast: true,
  };
 }
@@ -89,18 +89,19 @@ export async function handler(
 ) {
  const timestamp = getTimestamp(request.timestamp, request.body.payload);
  const ip =
-    path<string>(['properties', '__ip'], request.body.payload) ||
-    request.clientIp;
+    'properties' in request.body.payload &&
+    request.body.payload.properties?.__ip
+      ? (request.body.payload.properties.__ip as string)
+      : request.clientIp;
  const ua = request.headers['user-agent']!;
  const projectId = request.client?.projectId;

  if (!projectId) {
-    reply.status(400).send({
+    return reply.status(400).send({
      status: 400,
      error: 'Bad Request',
      message: 'Missing projectId',
    });
-    return;
  }

  const identity = getIdentity(request.body);
@@ -132,33 +133,7 @@ export async function handler(
          })
        : '';

-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-            previousDeviceId,
-            currentDeviceId,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
-      const promises = [
-        track({
-          payload: request.body.payload,
-          currentDeviceId,
-          previousDeviceId,
-          projectId,
-          geo,
-          headers: getStringHeaders(request.headers),
-          timestamp: timestamp.timestamp,
-          isTimestampFromThePast: timestamp.isTimestampFromThePast,
-        }),
-      ];
+      const promises = [];

      // If we have more than one property in the identity object, we should identify the user
      // Otherwise its only a profileId and we should not identify the user
@@ -173,23 +148,23 @@ export async function handler(
        );
      }

+      promises.push(
+        track({
+          payload: request.body.payload,
+          currentDeviceId,
+          previousDeviceId,
+          projectId,
+          geo,
+          headers: getStringHeaders(request.headers),
+          timestamp: timestamp.timestamp,
+          isTimestampFromThePast: timestamp.isTimestampFromThePast,
+        }),
+      );
+
      await Promise.all(promises);
      break;
    }
    case 'identify': {
-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
      const geo = await getGeoLocation(ip);
      await identify({
        payload: request.body.payload,
@@ -200,27 +175,13 @@ export async function handler(
      break;
    }
    case 'alias': {
-      reply.status(400).send({
+      return reply.status(400).send({
        status: 400,
        error: 'Bad Request',
        message: 'Alias is not supported',
      });
-      break;
    }
    case 'increment': {
-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
      await increment({
        payload: request.body.payload,
        projectId,
@@ -228,19 +189,6 @@ export async function handler(
      break;
    }
    case 'decrement': {
-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
      await decrement({
        payload: request.body.payload,
        projectId,
@@ -248,12 +196,11 @@ export async function handler(
      break;
    }
    default: {
-      reply.status(400).send({
+      return reply.status(400).send({
        status: 400,
        error: 'Bad Request',
        message: 'Invalid type',
      });
-      break;
    }
  }

@@ -276,7 +223,7 @@ async function track({
  projectId: string;
  geo: GeoLocation;
  headers: Record<string, string | undefined>;
-  timestamp: string;
+  timestamp: number;
  isTimestampFromThePast: boolean;
 }) {
  const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
@@ -285,8 +232,11 @@ async function track({
      ? `${projectId}:${payload.profileId}`
      : `${projectId}:${generateId()}`
    : currentDeviceId;
-  await eventsGroupQueue.add({
-    orderMs: new Date(timestamp).getTime(),
+  const jobId = [payload.name, timestamp, projectId, currentDeviceId, groupId]
+    .filter(Boolean)
+    .join('-');
+  await getEventsGroupQueueShard(groupId).add({
+    orderMs: timestamp,
    data: {
      projectId,
      headers,
@@ -295,11 +245,13 @@ async function track({
        timestamp,
        isTimestampFromThePast,
      },
+      uaInfo,
      geo,
      currentDeviceId,
      previousDeviceId,
    },
    groupId,
+    jobId,
  });
 }

@@ -322,8 +274,18 @@ async function identify({
    projectId,
    properties: {
      ...(payload.properties ?? {}),
-      ...(geo ?? {}),
-      ...uaInfo,
+      country: geo.country,
+      city: geo.city,
+      region: geo.region,
+      longitude: geo.longitude,
+      latitude: geo.latitude,
+      os: uaInfo.os,
+      os_version: uaInfo.osVersion,
+      browser: uaInfo.browser,
+      browser_version: uaInfo.browserVersion,
+      device: uaInfo.device,
+      brand: uaInfo.brand,
+      model: uaInfo.model,
    },
  });
 }
--- a/apps/api/src/hooks/duplicate.hook.ts
+++ b/apps/api/src/hooks/duplicate.hook.ts
@@ -0,0 +1,28 @@
+import { isDuplicatedEvent } from '@/utils/deduplicate';
+import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+
+export async function duplicateHook(
+  req: FastifyRequest<{
+    Body: PostEventPayload | TrackHandlerPayload;
+  }>,
+  reply: FastifyReply,
+) {
+  const ip = req.clientIp;
+  const origin = req.headers.origin;
+  const clientId = req.headers['openpanel-client-id'];
+  const shouldCheck = ip && origin && clientId;
+
+  const isDuplicate = shouldCheck
+    ? await isDuplicatedEvent({
+        ip,
+        origin,
+        payload: req.body,
+        projectId: clientId as string,
+      })
+    : false;
+
+  if (isDuplicate) {
+    return reply.status(200).send('Duplicate event');
+  }
+}
--- a/apps/api/src/hooks/fix.hook.ts
+++ b/apps/api/src/hooks/fix.hook.ts
@@ -1,16 +0,0 @@
-import type { FastifyRequest } from 'fastify';
-
-export async function fixHook(request: FastifyRequest) {
-  const ua = request.headers['user-agent'];
-  // Swift SDK issue: https://github.com/Openpanel-dev/swift-sdk/commit/d588fa761a36a33f3b78eb79d83bfd524e3c7144
-  if (ua) {
-    const regex = /OpenPanel\/(\d+\.\d+\.\d+)\sOpenPanel\/(\d+\.\d+\.\d+)/;
-    const match = ua.match(regex);
-    if (match) {
-      request.headers['user-agent'] = ua.replace(
-        regex,
-        `OpenPanel/${match[1]}`,
-      );
-    }
-  }
-}
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -28,7 +28,6 @@ import {
  liveness,
  readiness,
 } from './controllers/healthcheck.controller';
-import { fixHook } from './hooks/fix.hook';
 import { ipHook } from './hooks/ip.hook';
 import { requestIdHook } from './hooks/request-id.hook';
 import { requestLoggingHook } from './hooks/request-logging.hook';
@@ -125,7 +124,6 @@ const startServer = async () => {
    fastify.addHook('onRequest', requestIdHook);
    fastify.addHook('onRequest', timestampHook);
    fastify.addHook('onRequest', ipHook);
-    fastify.addHook('onRequest', fixHook);
    fastify.addHook('onResponse', requestLoggingHook);

    fastify.register(compress, {
--- a/apps/api/src/routes/event.router.ts
+++ b/apps/api/src/routes/event.router.ts
@@ -2,9 +2,11 @@ import * as controller from '@/controllers/event.controller';
 import type { FastifyPluginCallback } from 'fastify';

 import { clientHook } from '@/hooks/client.hook';
+import { duplicateHook } from '@/hooks/duplicate.hook';
 import { isBotHook } from '@/hooks/is-bot.hook';

 const eventRouter: FastifyPluginCallback = async (fastify) => {
+  fastify.addHook('preValidation', duplicateHook);
  fastify.addHook('preHandler', clientHook);
  fastify.addHook('preHandler', isBotHook);

--- a/apps/api/src/routes/track.router.ts
+++ b/apps/api/src/routes/track.router.ts
@@ -2,9 +2,11 @@ import { handler } from '@/controllers/track.controller';
 import type { FastifyPluginCallback } from 'fastify';

 import { clientHook } from '@/hooks/client.hook';
+import { duplicateHook } from '@/hooks/duplicate.hook';
 import { isBotHook } from '@/hooks/is-bot.hook';

 const trackRouter: FastifyPluginCallback = async (fastify) => {
+  fastify.addHook('preValidation', duplicateHook);
  fastify.addHook('preHandler', clientHook);
  fastify.addHook('preHandler', isBotHook);

--- a/apps/api/src/utils/auth.ts
+++ b/apps/api/src/utils/auth.ts
@@ -3,6 +3,7 @@ import type { FastifyRequest, RawRequestDefaultExpression } from 'fastify';
 import { verifyPassword } from '@openpanel/common/server';
 import type { IServiceClientWithProject } from '@openpanel/db';
 import { ClientType, getClientByIdCached } from '@openpanel/db';
+import { getCache } from '@openpanel/redis';
 import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
 import type {
  IProjectFilterIp,
@@ -135,7 +136,13 @@ export async function validateSdkRequest(
  }

  if (client.secret && clientSecret) {
-    if (await verifyPassword(clientSecret, client.secret)) {
+    const isVerified = await getCache(
+      `client:auth:${clientId}:${Buffer.from(clientSecret).toString('base64')}`,
+      60 * 5,
+      async () => await verifyPassword(clientSecret, client.secret!),
+      true,
+    );
+    if (isVerified) {
      return client;
    }
  }
--- a/apps/api/src/utils/deduplicate.ts
+++ b/apps/api/src/utils/deduplicate.ts
@@ -1,11 +1,14 @@
 import { getLock } from '@openpanel/redis';
 import fastJsonStableHash from 'fast-json-stable-hash';
-import type { FastifyReply } from 'fastify';

 export async function isDuplicatedEvent({
+  ip,
+  origin,
  payload,
  projectId,
 }: {
+  ip: string;
+  origin: string;
  payload: Record<string, any>;
  projectId: string;
 }) {
@@ -13,6 +16,8 @@ export async function isDuplicatedEvent({
    `fastify:deduplicate:${fastJsonStableHash.hash(
      {
        ...payload,
+        ip,
+        origin,
        projectId,
      },
      'md5',
@@ -27,24 +32,3 @@ export async function isDuplicatedEvent({

  return true;
 }
-
-export async function checkDuplicatedEvent({
-  reply,
-  payload,
-  projectId,
-}: {
-  reply: FastifyReply;
-  payload: Record<string, any>;
-  projectId: string;
-}) {
-  if (await isDuplicatedEvent({ payload, projectId })) {
-    reply.log.info('duplicated event', {
-      payload,
-      projectId,
-    });
-    reply.status(200).send('duplicated');
-    return true;
-  }
-
-  return false;
-}
--- a/apps/api/src/utils/graceful-shutdown.ts
+++ b/apps/api/src/utils/graceful-shutdown.ts
@@ -1,7 +1,7 @@
 import { ch, db } from '@openpanel/db';
 import {
  cronQueue,
-  eventsGroupQueue,
+  eventsGroupQueues,
  miscQueue,
  notificationQueue,
  sessionsQueue,
@@ -71,7 +71,7 @@ export async function shutdown(
  // Step 6: Close Bull queues (graceful shutdown of queue state)
  try {
    await Promise.all([
-      eventsGroupQueue.close(),
+      ...eventsGroupQueues.map((queue) => queue.close()),
      sessionsQueue.close(),
      cronQueue.close(),
      miscQueue.close(),
--- a/apps/start/package.json
+++ b/apps/start/package.json
@@ -19,7 +19,6 @@
  },
  "dependencies": {
    "@ai-sdk/react": "^1.2.5",
-    "@clickhouse/client": "^1.2.0",
    "@dnd-kit/core": "^6.3.1",
    "@dnd-kit/sortable": "^10.0.0",
    "@dnd-kit/utilities": "^3.2.2",
--- a/apps/start/src/components/realtime/realtime-geo.tsx
+++ b/apps/start/src/components/realtime/realtime-geo.tsx
@@ -75,7 +75,7 @@ export function RealtimeGeo({ projectId }: RealtimeGeoProps) {
          },
          {
            name: 'Events',
-            width: '84px',
+            width: '60px',
            render(item) {
              return (
                <div className="row gap-2 justify-end">
@@ -86,6 +86,19 @@ export function RealtimeGeo({ projectId }: RealtimeGeoProps) {
              );
            },
          },
+          {
+            name: 'Sessions',
+            width: '82px',
+            render(item) {
+              return (
+                <div className="row gap-2 justify-end">
+                  <span className="font-semibold">
+                    {number.short(item.unique_sessions)}
+                  </span>
+                </div>
+              );
+            },
+          },
        ]}
      />
    </div>
--- a/apps/start/src/components/realtime/realtime-paths.tsx
+++ b/apps/start/src/components/realtime/realtime-paths.tsx
@@ -82,7 +82,7 @@ export function RealtimePaths({ projectId }: RealtimePathsProps) {
          },
          {
            name: 'Events',
-            width: '84px',
+            width: '60px',
            render(item) {
              return (
                <div className="row gap-2 justify-end">
@@ -93,6 +93,19 @@ export function RealtimePaths({ projectId }: RealtimePathsProps) {
              );
            },
          },
+          {
+            name: 'Sessions',
+            width: '82px',
+            render(item) {
+              return (
+                <div className="row gap-2 justify-end">
+                  <span className="font-semibold">
+                    {number.short(item.unique_sessions)}
+                  </span>
+                </div>
+              );
+            },
+          },
        ]}
      />
    </div>
--- a/apps/start/src/components/realtime/realtime-referrals.tsx
+++ b/apps/start/src/components/realtime/realtime-referrals.tsx
@@ -65,7 +65,7 @@ export function RealtimeReferrals({ projectId }: RealtimeReferralsProps) {
          },
          {
            name: 'Events',
-            width: '84px',
+            width: '60px',
            render(item) {
              return (
                <div className="row gap-2 justify-end">
@@ -76,6 +76,19 @@ export function RealtimeReferrals({ projectId }: RealtimeReferralsProps) {
              );
            },
          },
+          {
+            name: 'Sessions',
+            width: '82px',
+            render(item) {
+              return (
+                <div className="row gap-2 justify-end">
+                  <span className="font-semibold">
+                    {number.short(item.unique_sessions)}
+                  </span>
+                </div>
+              );
+            },
+          },
        ]}
      />
    </div>
--- a/apps/start/src/components/ui/data-table/data-table-hooks.tsx
+++ b/apps/start/src/components/ui/data-table/data-table-hooks.tsx
@@ -4,8 +4,8 @@ import type {
  VisibilityState,
 } from '@tanstack/react-table';
 import { parseAsInteger, useQueryState } from 'nuqs';
-import { useState } from 'react';
-import { useLocalStorage } from 'usehooks-ts';
+import { useEffect, useState } from 'react';
+import { useLocalStorage, useReadLocalStorage } from 'usehooks-ts';

 export const useDataTablePagination = (pageSize = 10) => {
  const [page, setPage] = useQueryState(
@@ -22,6 +22,12 @@ export const useDataTablePagination = (pageSize = 10) => {
  return { page, setPage, state };
 };

+export const useReadColumnVisibility = (persistentKey: string) => {
+  return useReadLocalStorage<Record<string, boolean>>(
+    `@op:${persistentKey}-column-visibility`,
+  );
+};
+
 export const useDataTableColumnVisibility = <TData,>(
  columns: ColumnDef<TData>[],
  persistentKey: string,
@@ -43,6 +49,13 @@ export const useDataTableColumnVisibility = <TData,>(
    }, {} as VisibilityState),
  );

+  // somewhat hack
+  // Set initial column visibility,
+  // otherwise will not useReadColumnVisibility be updated
+  useEffect(() => {
+    setColumnVisibility(columnVisibility);
+  }, []);
+
  const [columnOrder, setColumnOrder] = useLocalStorage<string[]>(
    `@op:${persistentKey}-column-order`,
    columns.map((column) => column.id!),
--- a/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.conversions.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.conversions.tsx
@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import { useEventQueryNamesFilter } from '@/hooks/use-event-query-filters';
 import { useTRPC } from '@/integrations/trpc/react';
 import { useInfiniteQuery } from '@tanstack/react-query';
@@ -20,6 +21,7 @@ function Component() {
  );
  const [endDate, setEndDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
  const query = useInfiniteQuery(
    trpc.event.conversions.infiniteQueryOptions(
      {
@@ -27,6 +29,7 @@ function Component() {
        startDate: startDate || undefined,
        endDate: endDate || undefined,
        events: eventNames,
+        columnVisibility: columnVisibility ?? {},
      },
      {
        getNextPageParam: (lastPage) => lastPage.meta.next,
--- a/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx
@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
  useEventQueryFilters,
  useEventQueryNamesFilter,
@@ -21,6 +22,8 @@ function Component() {
  const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
  const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
+
  const query = useInfiniteQuery(
    trpc.event.events.infiniteQueryOptions(
      {
@@ -30,8 +33,10 @@ function Component() {
        profileId: '',
        startDate: startDate || undefined,
        endDate: endDate || undefined,
+        columnVisibility: columnVisibility ?? {},
      },
      {
+        enabled: columnVisibility !== null,
        getNextPageParam: (lastPage) => lastPage.meta.next,
      },
    ),
--- a/apps/start/src/routes/_app.$organizationId.$projectId.profiles.$profileId._tabs.events.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.profiles.$profileId._tabs.events.tsx
@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
  useEventQueryFilters,
  useEventQueryNamesFilter,
@@ -21,6 +22,7 @@ function Component() {
  const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
  const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
  const query = useInfiniteQuery(
    trpc.event.events.infiniteQueryOptions(
      {
@@ -30,8 +32,10 @@ function Component() {
        startDate: startDate || undefined,
        endDate: endDate || undefined,
        events: eventNames,
+        columnVisibility: columnVisibility ?? {},
      },
      {
+        enabled: columnVisibility !== null,
        getNextPageParam: (lastPage) => lastPage.meta.next,
      },
    ),
--- a/apps/start/src/routes/_app.$organizationId.$projectId.sessions_.$sessionId.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.sessions_.$sessionId.tsx
@@ -3,12 +3,11 @@ import FullPageLoadingState from '@/components/full-page-loading-state';
 import { PageContainer } from '@/components/page-container';
 import { PageHeader } from '@/components/page-header';
 import { SerieIcon } from '@/components/report-chart/common/serie-icon';
-import { useDataTablePagination } from '@/components/ui/data-table/data-table-hooks';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
  useEventQueryFilters,
  useEventQueryNamesFilter,
 } from '@/hooks/use-event-query-filters';
-import { useSearchQueryState } from '@/hooks/use-search-query-state';
 import { useTRPC } from '@/integrations/trpc/react';
 import { createProjectTitle } from '@/utils/title';
 import { useInfiniteQuery, useSuspenseQuery } from '@tanstack/react-query';
@@ -46,8 +45,6 @@ function Component() {
  const trpc = useTRPC();

  const LIMIT = 50;
-  const { page } = useDataTablePagination(LIMIT);
-  const { debouncedSearch } = useSearchQueryState();

  const { data: session } = useSuspenseQuery(
    trpc.session.byId.queryOptions({
@@ -60,7 +57,7 @@ function Component() {
  const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
  const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
-
+  const columnVisibility = useReadColumnVisibility('events');
  const query = useInfiniteQuery(
    trpc.event.events.infiniteQueryOptions(
      {
@@ -70,8 +67,10 @@ function Component() {
        events: eventNames,
        startDate: startDate || undefined,
        endDate: endDate || undefined,
+        columnVisibility: columnVisibility ?? {},
      },
      {
+        enabled: columnVisibility !== null,
        getNextPageParam: (lastPage) => lastPage.meta.next,
      },
    ),
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -11,8 +11,8 @@
    "typecheck": "tsc --noEmit"
  },
  "dependencies": {
-    "@bull-board/api": "6.13.1",
-    "@bull-board/express": "6.13.1",
+    "@bull-board/api": "6.14.0",
+    "@bull-board/express": "6.14.0",
    "@openpanel/common": "workspace:*",
    "@openpanel/db": "workspace:*",
    "@openpanel/email": "workspace:*",
@@ -22,9 +22,9 @@
    "@openpanel/importer": "workspace:*",
    "@openpanel/queue": "workspace:*",
    "@openpanel/redis": "workspace:*",
-    "bullmq": "^5.8.7",
+    "bullmq": "^5.63.0",
    "express": "^4.18.2",
-    "groupmq": "1.0.0-next.19",
+    "groupmq": "1.1.0-next.6",
    "prom-client": "^15.1.3",
    "ramda": "^0.29.1",
    "source-map-support": "^0.5.21",
--- a/apps/worker/src/boot-cron.ts
+++ b/apps/worker/src/boot-cron.ts
@@ -44,43 +44,30 @@ export async function bootCron() {
    });
  }

-  // Add repeatable jobs
-  for (const job of jobs) {
-    await cronQueue.add(
-      job.name,
-      {
-        type: job.type,
-        payload: undefined,
-      },
-      {
-        jobId: job.type,
-        repeat:
-          typeof job.pattern === 'number'
-            ? {
-                every: job.pattern,
-              }
-            : {
-                pattern: job.pattern,
-              },
-      },
-    );
+  logger.info('Updating cron jobs');
+
+  const jobSchedulers = await cronQueue.getJobSchedulers();
+  for (const jobScheduler of jobSchedulers) {
+    await cronQueue.removeJobScheduler(jobScheduler.key);
  }

-  // Remove outdated repeatable jobs
-  const repeatableJobs = await cronQueue.getRepeatableJobs();
-  for (const repeatableJob of repeatableJobs) {
-    const match = jobs.find(
-      (job) => `${job.name}:${job.type}:::${job.pattern}` === repeatableJob.key,
+  // Add repeatable jobs
+  for (const job of jobs) {
+    await cronQueue.upsertJobScheduler(
+      job.type,
+      typeof job.pattern === 'number'
+        ? {
+            every: job.pattern,
+          }
+        : {
+            pattern: job.pattern,
+          },
+      {
+        data: {
+          type: job.type,
+          payload: undefined,
+        },
+      },
    );
-    if (match) {
-      logger.info('Repeatable job exists', {
-        key: repeatableJob.key,
-      });
-    } else {
-      logger.info('Removing repeatable job', {
-        key: repeatableJob.key,
-      });
-      cronQueue.removeRepeatableByKey(repeatableJob.key);
-    }
  }
 }
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -2,9 +2,10 @@ import type { Queue, WorkerOptions } from 'bullmq';
 import { Worker } from 'bullmq';

 import {
+  EVENTS_GROUP_QUEUES_SHARDS,
  type EventsQueuePayloadIncomingEvent,
  cronQueue,
-  eventsGroupQueue,
+  eventsGroupQueues,
  importQueue,
  miscQueue,
  notificationQueue,
@@ -18,59 +19,179 @@ import { setTimeout as sleep } from 'node:timers/promises';
 import { Worker as GroupWorker } from 'groupmq';

 import { cronJob } from './jobs/cron';
-import { eventsJob } from './jobs/events';
-import { incomingEventPure } from './jobs/events.incoming-event';
+import { incomingEvent } from './jobs/events.incoming-event';
 import { importJob } from './jobs/import';
 import { miscJob } from './jobs/misc';
 import { notificationJob } from './jobs/notification';
 import { sessionsJob } from './jobs/sessions';
+import { eventsGroupJobDuration } from './metrics';
 import { logger } from './utils/logger';

 const workerOptions: WorkerOptions = {
  connection: getRedisQueue(),
 };

-export async function bootWorkers() {
-  const eventsGroupWorker = new GroupWorker<
-    EventsQueuePayloadIncomingEvent['payload']
-  >({
-    concurrency: Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '1', 10),
-    logger: queueLogger,
-    queue: eventsGroupQueue,
-    handler: async (job) => {
-      logger.info('processing event (group queue)', {
-        groupId: job.groupId,
-        timestamp: job.data.event.timestamp,
-      });
-      await incomingEventPure(job.data);
-    },
-  });
-  eventsGroupWorker.run();
-  const sessionsWorker = new Worker(
-    sessionsQueue.name,
-    sessionsJob,
-    workerOptions,
-  );
-  const cronWorker = new Worker(cronQueue.name, cronJob, workerOptions);
-  const notificationWorker = new Worker(
-    notificationQueue.name,
-    notificationJob,
-    workerOptions,
-  );
-  const miscWorker = new Worker(miscQueue.name, miscJob, workerOptions);
-  const importWorker = new Worker(importQueue.name, importJob, {
-    ...workerOptions,
-    concurrency: Number.parseInt(process.env.IMPORT_JOB_CONCURRENCY || '1', 10),
-  });
+type QueueName = string; // Can be: events, events_N (where N is 0 to shards-1), sessions, cron, notification, misc

-  const workers = [
-    sessionsWorker,
-    cronWorker,
-    notificationWorker,
-    miscWorker,
-    importWorker,
-    // eventsGroupWorker,
-  ];
+/**
+ * Parses the ENABLED_QUEUES environment variable and returns an array of queue names to start.
+ * If no env var is provided, returns all queues.
+ *
+ * Supported queue names:
+ * - events - All event shards (events_0, events_1, ..., events_N)
+ * - events_N - Individual event shard (where N is 0 to EVENTS_GROUP_QUEUES_SHARDS-1)
+ * - sessions, cron, notification, misc
+ */
+function getEnabledQueues(): QueueName[] {
+  const enabledQueuesEnv = process.env.ENABLED_QUEUES?.trim();
+
+  if (!enabledQueuesEnv) {
+    logger.info('No ENABLED_QUEUES specified, starting all queues', {
+      totalEventShards: EVENTS_GROUP_QUEUES_SHARDS,
+    });
+    return ['events', 'sessions', 'cron', 'notification', 'misc', 'import'];
+  }
+
+  const queues = enabledQueuesEnv
+    .split(',')
+    .map((q) => q.trim())
+    .filter(Boolean);
+
+  logger.info('Starting queues from ENABLED_QUEUES', {
+    queues,
+    totalEventShards: EVENTS_GROUP_QUEUES_SHARDS,
+  });
+  return queues;
+}
+
+/**
+ * Gets the concurrency setting for a queue from environment variables.
+ * Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., EVENTS_0_CONCURRENCY=32)
+ */
+function getConcurrencyFor(queueName: string, defaultValue = 1): number {
+  const envKey = `${queueName.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_CONCURRENCY`;
+  const value = process.env[envKey];
+
+  if (value) {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isNaN(parsed) && parsed > 0) {
+      return parsed;
+    }
+  }
+
+  return defaultValue;
+}
+
+export async function bootWorkers() {
+  const enabledQueues = getEnabledQueues();
+
+  const workers: (Worker | GroupWorker<any>)[] = [];
+
+  // Start event workers based on enabled queues
+  const eventQueuesToStart: number[] = [];
+
+  if (enabledQueues.includes('events')) {
+    // Start all event shards
+    for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
+      eventQueuesToStart.push(i);
+    }
+  } else {
+    // Start specific event shards (events_0, events_1, etc.)
+    for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
+      if (enabledQueues.includes(`events_${i}`)) {
+        eventQueuesToStart.push(i);
+      }
+    }
+  }
+
+  for (const index of eventQueuesToStart) {
+    const queue = eventsGroupQueues[index];
+    if (!queue) continue;
+
+    const queueName = `events_${index}`;
+    const concurrency = getConcurrencyFor(
+      queueName,
+      Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10),
+    );
+
+    const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
+      queue,
+      concurrency,
+      logger: queueLogger,
+      blockingTimeoutSec: Number.parseFloat(
+        process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',
+      ),
+      handler: async (job) => {
+        return await incomingEvent(job.data);
+      },
+    });
+
+    worker.run();
+    workers.push(worker);
+    logger.info(`Started worker for ${queueName}`, { concurrency });
+  }
+
+  // Start sessions worker
+  if (enabledQueues.includes('sessions')) {
+    const concurrency = getConcurrencyFor('sessions');
+    const sessionsWorker = new Worker(sessionsQueue.name, sessionsJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(sessionsWorker);
+    logger.info('Started worker for sessions', { concurrency });
+  }
+
+  // Start cron worker
+  if (enabledQueues.includes('cron')) {
+    const concurrency = getConcurrencyFor('cron');
+    const cronWorker = new Worker(cronQueue.name, cronJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(cronWorker);
+    logger.info('Started worker for cron', { concurrency });
+  }
+
+  // Start notification worker
+  if (enabledQueues.includes('notification')) {
+    const concurrency = getConcurrencyFor('notification');
+    const notificationWorker = new Worker(
+      notificationQueue.name,
+      notificationJob,
+      { ...workerOptions, concurrency },
+    );
+    workers.push(notificationWorker);
+    logger.info('Started worker for notification', { concurrency });
+  }
+
+  // Start misc worker
+  if (enabledQueues.includes('misc')) {
+    const concurrency = getConcurrencyFor('misc');
+    const miscWorker = new Worker(miscQueue.name, miscJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(miscWorker);
+    logger.info('Started worker for misc', { concurrency });
+  }
+
+  // Start import worker
+  if (enabledQueues.includes('import')) {
+    const concurrency = getConcurrencyFor('import');
+    const importWorker = new Worker(importQueue.name, importJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(importWorker);
+    logger.info('Started worker for import', { concurrency });
+  }
+
+  if (workers.length === 0) {
+    logger.warn(
+      'No workers started. Check ENABLED_QUEUES environment variable.',
+    );
+  }

  workers.forEach((worker) => {
    (worker as Worker).on('error', (error) => {
@@ -94,6 +215,13 @@ export async function bootWorkers() {

    (worker as Worker).on('failed', (job) => {
      if (job) {
+        if (job.processedOn && job.finishedOn) {
+          const elapsed = job.finishedOn - job.processedOn;
+          eventsGroupJobDuration.observe(
+            { name: worker.name, status: 'failed' },
+            elapsed,
+          );
+        }
        logger.error('job failed', {
          jobId: job.id,
          worker: worker.name,
@@ -106,15 +234,18 @@ export async function bootWorkers() {

    (worker as Worker).on('completed', (job) => {
      if (job) {
-        logger.info('job completed', {
-          jobId: job.id,
-          worker: worker.name,
-          data: job.data,
-          elapsed:
-            job.processedOn && job.finishedOn
-              ? job.finishedOn - job.processedOn
-              : undefined,
-        });
+        if (job.processedOn && job.finishedOn) {
+          const elapsed = job.finishedOn - job.processedOn;
+          logger.info('job completed', {
+            jobId: job.id,
+            worker: worker.name,
+            elapsed,
+          });
+          eventsGroupJobDuration.observe(
+            { name: worker.name, status: 'success' },
+            elapsed,
+          );
+        }
      }
    });

@@ -135,8 +266,14 @@ export async function bootWorkers() {
    });
    try {
      const time = performance.now();
-      await waitForQueueToEmpty(cronQueue);
+
+      // Wait for cron queue to empty if it's running
+      if (enabledQueues.includes('cron')) {
+        await waitForQueueToEmpty(cronQueue);
+      }
+
      await Promise.all(workers.map((worker) => worker.close()));
+
      logger.info('workers closed successfully', {
        elapsed: performance.now() - time,
      });
@@ -155,15 +292,7 @@ export async function bootWorkers() {
  ['uncaughtException', 'unhandledRejection', 'SIGTERM', 'SIGINT'].forEach(
    (evt) => {
      process.on(evt, (code) => {
-        if (process.env.NODE_ENV === 'production') {
-          exitHandler(evt, code);
-        } else {
-          logger.info('Shutting down for development', {
-            event: evt,
-            code,
-          });
-          process.exit(0);
-        }
+        exitHandler(evt, code);
      });
    },
  );
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -4,7 +4,7 @@ import { ExpressAdapter } from '@bull-board/express';
 import { createInitialSalts } from '@openpanel/db';
 import {
  cronQueue,
-  eventsGroupQueue,
+  eventsGroupQueues,
  importQueue,
  miscQueue,
  notificationQueue,
@@ -34,7 +34,9 @@ async function start() {
    serverAdapter.setBasePath('/');
    createBullBoard({
      queues: [
-        new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
+        ...eventsGroupQueues.map(
+          (queue) => new BullBoardGroupMQAdapter(queue) as any,
+        ),
        new BullMQAdapter(sessionsQueue),
        new BullMQAdapter(cronQueue),
        new BullMQAdapter(notificationQueue),
--- a/apps/worker/src/jobs/events.create-session-end.ts
+++ b/apps/worker/src/jobs/events.create-session-end.ts
@@ -1,13 +1,13 @@
 import type { Job } from 'bullmq';

 import { logger as baseLogger } from '@/utils/logger';
-import { getTime } from '@openpanel/common';
 import {
  type IClickhouseSession,
  type IServiceCreateEventPayload,
  type IServiceEvent,
  TABLE_NAMES,
  checkNotificationRulesForSessionEnd,
+  convertClickhouseDateToJs,
  createEvent,
  eventBuffer,
  formatClickhouseDate,
@@ -65,10 +65,9 @@ export async function createSessionEnd(
  const logger = baseLogger.child({
    payload,
    jobId: job.id,
-    reqId: payload.properties?.__reqId ?? 'unknown',
  });

-  logger.info('Processing session end job');
+  logger.debug('Processing session end job');

  const session = await sessionBuffer.getExistingSession(payload.sessionId);

@@ -77,7 +76,7 @@ export async function createSessionEnd(
  }

  try {
-    handleSessionEndNotifications({
+    await handleSessionEndNotifications({
      session,
      payload,
    });
@@ -103,7 +102,9 @@ export async function createSessionEnd(
    name: 'session_end',
    duration: session.duration ?? 0,
    path: lastScreenView?.path ?? '',
-    createdAt: new Date(getTime(session.ended_at) + 1000),
+    createdAt: new Date(
+      convertClickhouseDateToJs(session.ended_at).getTime() + 100,
+    ),
    profileId: lastScreenView?.profileId || payload.profileId,
  });
 }
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -18,9 +18,7 @@ import {
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
-import type { Job } from 'bullmq';
 import * as R from 'ramda';
-import { omit } from 'ramda';
 import { v4 as uuid } from 'uuid';

 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
@@ -33,10 +31,9 @@ const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>

 async function createEventAndNotify(
  payload: IServiceCreateEventPayload,
-  jobData: Job<EventsQueuePayloadIncomingEvent>['data']['payload'],
  logger: ILogger,
 ) {
-  logger.info('Creating event', { event: payload, jobData });
+  logger.info('Creating event', { event: payload });
  const [event] = await Promise.all([
    createEvent(payload),
    checkNotificationRulesForEvent(payload).catch(() => {}),
@@ -45,16 +42,7 @@ async function createEventAndNotify(
 }

 export async function incomingEvent(
-  job: Job<EventsQueuePayloadIncomingEvent>,
-  token?: string,
-) {
-  return incomingEventPure(job.data.payload, job, token);
-}
-
-export async function incomingEventPure(
  jobPayload: EventsQueuePayloadIncomingEvent['payload'],
-  job?: Job<EventsQueuePayloadIncomingEvent>,
-  token?: string,
 ) {
  const {
    geo,
@@ -63,6 +51,7 @@ export async function incomingEventPure(
    projectId,
    currentDeviceId,
    previousDeviceId,
+    uaInfo: _uaInfo,
  } = jobPayload;
  const properties = body.properties ?? {};
  const reqId = headers['request-id'] ?? 'unknown';
@@ -93,18 +82,17 @@ export async function incomingEventPure(
  const userAgent = headers['user-agent'];
  const sdkName = headers['openpanel-sdk-name'];
  const sdkVersion = headers['openpanel-sdk-version'];
-  const uaInfo = parseUserAgent(userAgent, properties);
+  // TODO: Remove both user-agent and parseUserAgent
+  const uaInfo = _uaInfo ?? parseUserAgent(userAgent, properties);

  const baseEvent = {
    name: body.name,
    profileId,
    projectId,
-    properties: omit(GLOBAL_PROPERTIES, {
+    properties: R.omit(GLOBAL_PROPERTIES, {
      ...properties,
-      __user_agent: userAgent,
      __hash: hash,
      __query: query,
-      __reqId: reqId,
    }),
    createdAt,
    duration: 0,
@@ -161,7 +149,7 @@ export async function incomingEventPure(
      origin: screenView?.origin ?? baseEvent.origin,
    };

-    return createEventAndNotify(payload as IServiceEvent, jobPayload, logger);
+    return createEventAndNotify(payload as IServiceEvent, logger);
  }

  const sessionEnd = await getSessionEnd({
@@ -197,7 +185,7 @@ export async function incomingEventPure(
    });
  }

-  const event = await createEventAndNotify(payload, jobPayload, logger);
+  const event = await createEventAndNotify(payload, logger);

  if (!sessionEnd) {
    logger.info('Creating session end job', { event: payload });
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -1,6 +1,9 @@
 import { type IServiceEvent, createEvent } from '@openpanel/db';
 import { eventBuffer } from '@openpanel/db';
-import { sessionsQueue } from '@openpanel/queue';
+import {
+  type EventsQueuePayloadIncomingEvent,
+  sessionsQueue,
+} from '@openpanel/queue';
 import type { Job } from 'bullmq';
 import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest';
 import { incomingEvent } from './events.incoming-event';
@@ -32,6 +35,28 @@ const geo = {
  latitude: 0,
 };

+const uaInfo: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
+  isServer: false,
+  device: 'desktop',
+  os: 'Windows',
+  osVersion: '10',
+  browser: 'Chrome',
+  browserVersion: '91.0.4472.124',
+  brand: '',
+  model: '',
+};
+
+const uaInfoServer: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
+  isServer: true,
+  device: 'server',
+  os: '',
+  osVersion: '',
+  browser: '',
+  browserVersion: '',
+  brand: '',
+  model: '',
+};
+
 describe('incomingEvent', () => {
  beforeEach(() => {
    vi.clearAllMocks();
@@ -41,31 +66,29 @@ describe('incomingEvent', () => {
    const spySessionsQueueAdd = vi.spyOn(sessionsQueue, 'add');
    const timestamp = new Date();
    // Mock job data
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'test_event',
-          timestamp: timestamp.toISOString(),
-          properties: { __path: 'https://example.com/test' },
-        },
-        headers: {
-          'request-id': '123',
-          'user-agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-          'openpanel-sdk-name': 'web',
-          'openpanel-sdk-version': '1.0.0',
-        },
-        projectId,
-        currentDeviceId,
-        previousDeviceId,
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'test_event',
+        timestamp: timestamp.toISOString(),
+        isTimestampFromThePast: false,
+        properties: { __path: 'https://example.com/test' },
      },
+      uaInfo,
+      headers: {
+        'request-id': '123',
+        'user-agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'openpanel-sdk-name': 'web',
+        'openpanel-sdk-version': '1.0.0',
+      },
+      projectId,
+      currentDeviceId,
+      previousDeviceId,
    };

-    const job = { data: jobData } as Job;
-
    // Execute the job
-    await incomingEvent(job);
+    await incomingEvent(jobData);

    const event = {
      name: 'test_event',
@@ -78,8 +101,6 @@ describe('incomingEvent', () => {
      properties: {
        __hash: undefined,
        __query: undefined,
-        __user_agent: jobData.payload.headers['user-agent'],
-        __reqId: jobData.payload.headers['request-id'],
      },
      createdAt: timestamp,
      country: 'US',
@@ -92,16 +113,16 @@ describe('incomingEvent', () => {
      browser: 'Chrome',
      browserVersion: '91.0.4472.124',
      device: 'desktop',
-      brand: undefined,
-      model: undefined,
+      brand: '',
+      model: '',
      duration: 0,
      path: '/test',
      origin: 'https://example.com',
      referrer: '',
      referrerName: '',
      referrerType: '',
-      sdkName: jobData.payload.headers['openpanel-sdk-name'],
-      sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
+      sdkName: jobData.headers['openpanel-sdk-name'],
+      sdkVersion: jobData.headers['openpanel-sdk-version'],
    };

    expect(spySessionsQueueAdd).toHaveBeenCalledWith(
@@ -135,29 +156,27 @@ describe('incomingEvent', () => {

    const timestamp = new Date();
    // Mock job data
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'test_event',
-          timestamp: timestamp.toISOString(),
-          properties: { __path: 'https://example.com/test' },
-        },
-        headers: {
-          'request-id': '123',
-          'user-agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-          'openpanel-sdk-name': 'web',
-          'openpanel-sdk-version': '1.0.0',
-        },
-        projectId,
-        currentDeviceId,
-        previousDeviceId,
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'test_event',
+        timestamp: timestamp.toISOString(),
+        properties: { __path: 'https://example.com/test' },
+        isTimestampFromThePast: false,
      },
+      headers: {
+        'request-id': '123',
+        'user-agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'openpanel-sdk-name': 'web',
+        'openpanel-sdk-version': '1.0.0',
+      },
+      uaInfo,
+      projectId,
+      currentDeviceId,
+      previousDeviceId,
    };

-    const job = { data: jobData } as Job;
-
    const changeDelay = vi.fn();
    const updateData = vi.fn();
    spySessionsQueueGetJob.mockResolvedValueOnce({
@@ -175,7 +194,7 @@ describe('incomingEvent', () => {
      },
    } as Partial<Job> as Job);
    // Execute the job
-    await incomingEvent(job);
+    await incomingEvent(jobData);

    const event = {
      name: 'test_event',
@@ -186,8 +205,6 @@ describe('incomingEvent', () => {
      properties: {
        __hash: undefined,
        __query: undefined,
-        __user_agent: jobData.payload.headers['user-agent'],
-        __reqId: jobData.payload.headers['request-id'],
      },
      createdAt: timestamp,
      country: 'US',
@@ -200,16 +217,16 @@ describe('incomingEvent', () => {
      browser: 'Chrome',
      browserVersion: '91.0.4472.124',
      device: 'desktop',
-      brand: undefined,
-      model: undefined,
+      brand: '',
+      model: '',
      duration: 0,
      path: '/test',
      origin: 'https://example.com',
      referrer: '',
      referrerName: '',
      referrerType: '',
-      sdkName: jobData.payload.headers['openpanel-sdk-name'],
-      sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
+      sdkName: jobData.headers['openpanel-sdk-name'],
+      sdkVersion: jobData.headers['openpanel-sdk-version'],
    };

    expect(spySessionsQueueAdd).toHaveBeenCalledTimes(0);
@@ -220,29 +237,27 @@ describe('incomingEvent', () => {

  it('should handle server events (with existing screen view)', async () => {
    const timestamp = new Date();
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'server_event',
-          timestamp: timestamp.toISOString(),
-          properties: { custom_property: 'test_value' },
-          profileId: 'profile-123',
-        },
-        headers: {
-          'user-agent': 'OpenPanel Server/1.0',
-          'openpanel-sdk-name': 'server',
-          'openpanel-sdk-version': '1.0.0',
-          'request-id': '123',
-        },
-        projectId,
-        currentDeviceId: '',
-        previousDeviceId: '',
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'server_event',
+        timestamp: timestamp.toISOString(),
+        properties: { custom_property: 'test_value' },
+        profileId: 'profile-123',
+        isTimestampFromThePast: false,
      },
+      headers: {
+        'user-agent': 'OpenPanel Server/1.0',
+        'openpanel-sdk-name': 'server',
+        'openpanel-sdk-version': '1.0.0',
+        'request-id': '123',
+      },
+      projectId,
+      currentDeviceId: '',
+      previousDeviceId: '',
+      uaInfo: uaInfoServer,
    };

-    const job = { data: jobData } as Job;
-
    const mockLastScreenView = {
      deviceId: 'last-device-123',
      sessionId: 'last-session-456',
@@ -268,7 +283,7 @@ describe('incomingEvent', () => {
      mockLastScreenView as IServiceEvent,
    );

-    await incomingEvent(job);
+    await incomingEvent(jobData);

    expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
      name: 'server_event',
@@ -278,8 +293,6 @@ describe('incomingEvent', () => {
      projectId,
      properties: {
        custom_property: 'test_value',
-        __user_agent: 'OpenPanel Server/1.0',
-        __reqId: '123',
        __hash: undefined,
        __query: undefined,
      },
@@ -311,33 +324,31 @@ describe('incomingEvent', () => {

  it('should handle server events (without existing screen view)', async () => {
    const timestamp = new Date();
-    const jobData = {
-      payload: {
-        geo,
-        event: {
-          name: 'server_event',
-          timestamp: timestamp.toISOString(),
-          properties: { custom_property: 'test_value' },
-          profileId: 'profile-123',
-        },
-        headers: {
-          'user-agent': 'OpenPanel Server/1.0',
-          'openpanel-sdk-name': 'server',
-          'openpanel-sdk-version': '1.0.0',
-          'request-id': '123',
-        },
-        projectId,
-        currentDeviceId: '',
-        previousDeviceId: '',
+    const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
+      geo,
+      event: {
+        name: 'server_event',
+        timestamp: timestamp.toISOString(),
+        properties: { custom_property: 'test_value' },
+        profileId: 'profile-123',
+        isTimestampFromThePast: false,
      },
+      headers: {
+        'user-agent': 'OpenPanel Server/1.0',
+        'openpanel-sdk-name': 'server',
+        'openpanel-sdk-version': '1.0.0',
+        'request-id': '123',
+      },
+      projectId,
+      currentDeviceId: '',
+      previousDeviceId: '',
+      uaInfo: uaInfoServer,
    };

-    const job = { data: jobData } as Job;
-
    // Mock getLastScreenView to return null
    vi.mocked(eventBuffer.getLastScreenView).mockResolvedValueOnce(null);

-    await incomingEvent(job);
+    await incomingEvent(jobData);

    expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
      name: 'server_event',
@@ -347,8 +358,6 @@ describe('incomingEvent', () => {
      projectId,
      properties: {
        custom_property: 'test_value',
-        __user_agent: 'OpenPanel Server/1.0',
-        __reqId: '123',
        __hash: undefined,
        __query: undefined,
      },
--- a/apps/worker/src/jobs/events.ts
+++ b/apps/worker/src/jobs/events.ts
@@ -1,15 +0,0 @@
-import type { Job } from 'bullmq';
-
-import type {
-  EventsQueuePayload,
-  EventsQueuePayloadIncomingEvent,
-} from '@openpanel/queue';
-
-import { incomingEvent } from './events.incoming-event';
-
-export async function eventsJob(job: Job<EventsQueuePayload>, token?: string) {
-  return await incomingEvent(
-    job as Job<EventsQueuePayloadIncomingEvent>,
-    token,
-  );
-}
--- a/apps/worker/src/metrics.ts
+++ b/apps/worker/src/metrics.ts
@@ -2,23 +2,32 @@ import client from 'prom-client';

 import {
  botBuffer,
-  db,
  eventBuffer,
  profileBuffer,
  sessionBuffer,
 } from '@openpanel/db';
-import { cronQueue, eventsGroupQueue, sessionsQueue } from '@openpanel/queue';
+import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';

 const Registry = client.Registry;

 export const register = new Registry();

-const queues = [sessionsQueue, cronQueue, eventsGroupQueue];
+const queues = [sessionsQueue, cronQueue, ...eventsGroupQueues];
+
+// Histogram to track job processing time for eventsGroupQueues
+export const eventsGroupJobDuration = new client.Histogram({
+  name: 'job_duration_ms',
+  help: 'Duration of job processing (in ms)',
+  labelNames: ['name', 'status'],
+  buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
+});
+
+register.registerMetric(eventsGroupJobDuration);

 queues.forEach((queue) => {
  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_active_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
      help: 'Active count',
      async collect() {
        const metric = await queue.getActiveCount();
@@ -29,7 +38,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_delayed_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
      help: 'Delayed count',
      async collect() {
        const metric = await queue.getDelayedCount();
@@ -40,7 +49,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_failed_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
      help: 'Failed count',
      async collect() {
        const metric = await queue.getFailedCount();
@@ -51,7 +60,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_completed_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
      help: 'Completed count',
      async collect() {
        const metric = await queue.getCompletedCount();
@@ -62,7 +71,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_waiting_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
      help: 'Waiting count',
      async collect() {
        const metric = await queue.getWaitingCount();
--- a/apps/worker/src/utils/session-handler.ts
+++ b/apps/worker/src/utils/session-handler.ts
@@ -113,13 +113,12 @@ export async function getSessionEndJob(args: {
  } | null> {
    const state = await job.getState();
    if (state !== 'delayed') {
-      logger.info(`[session-handler] Session end job is in "${state}" state`, {
+      logger.debug(`[session-handler] Session end job is in "${state}" state`, {
        state,
        retryCount,
        jobTimestamp: new Date(job.timestamp).toISOString(),
        jobDelta: Date.now() - job.timestamp,
        jobId: job.id,
-        reqId: job.data.payload.properties?.__reqId ?? 'unknown',
        payload: job.data.payload,
      });
    }