fix: performance related fixes

2025-10-22 12:29:56 +02:00
parent 8bb0c87ec9
commit 1285ad85a2
60 changed files with 4264 additions and 2959 deletions
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -13,11 +13,11 @@
  "dependencies": {
    "@ai-sdk/anthropic": "^1.2.10",
    "@ai-sdk/openai": "^1.3.12",
-    "@fastify/compress": "^8.0.1",
+    "@fastify/compress": "^8.1.0",
    "@fastify/cookie": "^11.0.2",
-    "@fastify/cors": "^11.0.0",
-    "@fastify/rate-limit": "^10.2.2",
-    "@fastify/websocket": "^11.0.2",
+    "@fastify/cors": "^11.1.0",
+    "@fastify/rate-limit": "^10.3.0",
+    "@fastify/websocket": "^11.2.0",
    "@node-rs/argon2": "^2.0.2",
    "@openpanel/auth": "workspace:^",
    "@openpanel/common": "workspace:*",
@@ -35,10 +35,10 @@
    "@trpc/server": "^11.6.0",
    "ai": "^4.2.10",
    "fast-json-stable-hash": "^1.0.3",
-    "fastify": "^5.2.1",
+    "fastify": "^5.6.1",
    "fastify-metrics": "^12.1.0",
    "fastify-raw-body": "^5.0.0",
-    "groupmq": "1.0.0-next.19",
+    "groupmq": "1.1.0-next.5",
    "jsonwebtoken": "^9.0.2",
    "ramda": "^0.29.1",
    "sharp": "^0.33.5",
--- a/apps/api/scripts/get-bots.ts
+++ b/apps/api/scripts/get-bots.ts
@@ -7,6 +7,23 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 import yaml from 'js-yaml';

+// Regex special characters that indicate we need actual regex
+const regexSpecialChars = /[|^$.*+?(){}\[\]\\]/;
+
+function transformBots(bots: any[]): any[] {
+  return bots.map((bot) => {
+    const { regex, ...rest } = bot;
+    const hasRegexChars = regexSpecialChars.test(regex);
+
+    if (hasRegexChars) {
+      // Keep as regex
+      return { regex, ...rest };
+    }
+    // Convert to includes
+    return { includes: regex, ...rest };
+  });
+}
+
 async function main() {
  // Get document, or throw exception on error
  try {
@@ -14,6 +31,9 @@ async function main() {
      'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
    ).then((res) => res.text());

+    const parsedData = yaml.load(data) as any[];
+    const transformedBots = transformBots(parsedData);
+
    fs.writeFileSync(
      path.resolve(__dirname, '../src/bots/bots.ts'),
      [
@@ -21,11 +41,20 @@ async function main() {
        '',
        '// The data is fetch from device-detector https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
        '',
-        `const bots = ${JSON.stringify(yaml.load(data))} as const;`,
+        `const bots = ${JSON.stringify(transformedBots, null, 2)} as const;`,
        'export default bots;',
+        '',
      ].join('\n'),
      'utf-8',
    );
+
+    console.log(
+      `✅ Generated bots.ts with ${transformedBots.length} bot entries`,
+    );
+    const regexCount = transformedBots.filter((b) => 'regex' in b).length;
+    const includesCount = transformedBots.filter((b) => 'includes' in b).length;
+    console.log(`   - ${includesCount} simple string matches (includes)`);
+    console.log(`   - ${regexCount} regex patterns`);
  } catch (e) {
    console.log(e);
  }
--- a/apps/api/src/bots/bots.ts
+++ b/apps/api/src/bots/bots.ts
--- a/apps/api/src/bots/index.ts
+++ b/apps/api/src/bots/index.ts
@@ -1,19 +1,72 @@
 import bots from './bots';

-export function isBot(ua: string) {
-  const res = bots.find((bot) => {
-    if (new RegExp(bot.regex).test(ua)) {
-      return true;
-    }
-    return false;
-  });
+// Pre-compile regex patterns at module load time
+const compiledBots = bots.map((bot) => {
+  if ('regex' in bot) {
+    return {
+      ...bot,
+      compiledRegex: new RegExp(bot.regex),
+    };
+  }
+  return bot;
+});

-  if (!res) {
-    return null;
+const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
+const includesBots = compiledBots.filter((bot) => 'includes' in bot);
+
+// Common legitimate browser patterns - if UA matches these, it's very likely a real browser
+// This provides ultra-fast early exit for ~95% of real traffic
+const legitimateBrowserPatterns = [
+  'Mozilla/5.0', // Nearly all modern browsers
+  'Chrome/', // Chrome/Chromium browsers
+  'Safari/', // Safari and Chrome-based browsers
+  'Firefox/', // Firefox
+  'Edg/', // Edge
+];
+
+const mobilePatterns = ['iPhone', 'Android', 'iPad'];
+
+const desktopOSPatterns = ['Windows NT', 'Macintosh', 'X11; Linux'];
+
+export function isBot(ua: string) {
+  // Ultra-fast early exit: check if this looks like a legitimate browser
+  // Real browsers typically have Mozilla/5.0 + browser name + OS
+  if (ua.includes('Mozilla/5.0')) {
+    // Check for browser signature
+    const hasBrowser = legitimateBrowserPatterns.some((pattern) =>
+      ua.includes(pattern),
+    );
+
+    // Check for OS signature (mobile or desktop)
+    const hasOS =
+      mobilePatterns.some((pattern) => ua.includes(pattern)) ||
+      desktopOSPatterns.some((pattern) => ua.includes(pattern));
+
+    // If it has Mozilla/5.0, a known browser, and an OS, it's very likely legitimate
+    if (hasBrowser && hasOS) {
+      return null;
+    }
  }

-  return {
-    name: res.name,
-    type: 'category' in res ? res.category : 'Unknown',
-  };
+  // Check simple string patterns first (fast)
+  for (const bot of includesBots) {
+    if (ua.includes(bot.includes)) {
+      return {
+        name: bot.name,
+        type: 'category' in bot ? bot.category : 'Unknown',
+      };
+    }
+  }
+
+  // Check regex patterns (slower)
+  for (const bot of regexBots) {
+    if (bot.compiledRegex.test(ua)) {
+      return {
+        name: bot.name,
+        type: 'category' in bot ? bot.category : 'Unknown',
+      };
+    }
+  }
+
+  return null;
 }
--- a/apps/api/src/controllers/event.controller.ts
+++ b/apps/api/src/controllers/event.controller.ts
@@ -2,10 +2,9 @@ import type { FastifyReply, FastifyRequest } from 'fastify';

 import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
 import { getSalts } from '@openpanel/db';
-import { eventsGroupQueue } from '@openpanel/queue';
+import { getEventsGroupQueueShard } from '@openpanel/queue';
 import type { PostEventPayload } from '@openpanel/sdk';

-import { checkDuplicatedEvent } from '@/utils/deduplicate';
 import { generateId } from '@openpanel/common';
 import { getGeoLocation } from '@openpanel/geo';
 import { getStringHeaders, getTimestamp } from './track.controller';
@@ -44,28 +43,22 @@ export async function postEvent(
    ua,
  });

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-        timestamp,
-        previousDeviceId,
-        currentDeviceId,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  const uaInfo = parseUserAgent(ua, request.body?.properties);
  const groupId = uaInfo.isServer
    ? request.body?.profileId
      ? `${projectId}:${request.body?.profileId}`
      : `${projectId}:${generateId()}`
    : currentDeviceId;
-  await eventsGroupQueue.add({
+  const jobId = [
+    request.body.name,
+    timestamp,
+    projectId,
+    currentDeviceId,
+    groupId,
+  ]
+    .filter(Boolean)
+    .join('-');
+  await getEventsGroupQueueShard(groupId).add({
    orderMs: new Date(timestamp).getTime(),
    data: {
      projectId,
@@ -75,11 +68,13 @@ export async function postEvent(
        timestamp,
        isTimestampFromThePast,
      },
+      uaInfo,
      geo,
      currentDeviceId,
      previousDeviceId,
    },
    groupId,
+    jobId,
  });

  reply.status(202).send('ok');
--- a/apps/api/src/controllers/live.controller.ts
+++ b/apps/api/src/controllers/live.controller.ts
@@ -4,7 +4,7 @@ import superjson from 'superjson';
 import type { WebSocket } from '@fastify/websocket';
 import {
  eventBuffer,
-  getProfileByIdCached,
+  getProfileById,
  transformMinimalEvent,
 } from '@openpanel/db';
 import { setSuperJson } from '@openpanel/json';
@@ -92,10 +92,7 @@ export async function wsProjectEvents(
    type,
    async (event) => {
      if (event.projectId === params.projectId) {
-        const profile = await getProfileByIdCached(
-          event.profileId,
-          event.projectId,
-        );
+        const profile = await getProfileById(event.profileId, event.projectId);
        socket.send(
          superjson.stringify(
            access
--- a/apps/api/src/controllers/misc.controller.ts
+++ b/apps/api/src/controllers/misc.controller.ts
@@ -132,7 +132,7 @@ async function processImage(
 ): Promise<Buffer> {
  // If it's an ICO file, just return it as-is (no conversion needed)
  if (originalUrl && isIcoFile(originalUrl, contentType)) {
-    logger.info('Serving ICO file directly', {
+    logger.debug('Serving ICO file directly', {
      originalUrl,
      bufferSize: buffer.length,
    });
@@ -140,7 +140,7 @@ async function processImage(
  }

  if (originalUrl && isSvgFile(originalUrl, contentType)) {
-    logger.info('Serving SVG file directly', {
+    logger.debug('Serving SVG file directly', {
      originalUrl,
      bufferSize: buffer.length,
    });
@@ -149,7 +149,7 @@ async function processImage(

  // If buffer isnt to big just return it as well
  if (buffer.length < 5000) {
-    logger.info('Serving image directly without processing', {
+    logger.debug('Serving image directly without processing', {
      originalUrl,
      bufferSize: buffer.length,
    });
@@ -193,7 +193,7 @@ async function processOgImage(
 ): Promise<Buffer> {
  // If buffer is small enough, return it as-is
  if (buffer.length < 10000) {
-    logger.info('Serving OG image directly without processing', {
+    logger.debug('Serving OG image directly without processing', {
      originalUrl,
      bufferSize: buffer.length,
    });
--- a/apps/api/src/controllers/profile.controller.ts
+++ b/apps/api/src/controllers/profile.controller.ts
@@ -1,7 +1,6 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { assocPath, pathOr } from 'ramda';

-import { checkDuplicatedEvent, isDuplicatedEvent } from '@/utils/deduplicate';
 import { parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, upsertProfile } from '@openpanel/db';
 import { getGeoLocation } from '@openpanel/geo';
@@ -16,41 +15,39 @@ export async function updateProfile(
  }>,
  reply: FastifyReply,
 ) {
-  const { profileId, properties, ...rest } = request.body;
+  const payload = request.body;
  const projectId = request.client!.projectId;
  if (!projectId) {
    return reply.status(400).send('No projectId');
  }
  const ip = request.clientIp;
  const ua = request.headers['user-agent']!;
-  const uaInfo = parseUserAgent(ua, properties);
+  const uaInfo = parseUserAgent(ua, payload.properties);
  const geo = await getGeoLocation(ip);

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  await upsertProfile({
-    id: profileId,
+    ...payload,
+    id: payload.profileId,
    isExternal: true,
    projectId,
    properties: {
-      ...(properties ?? {}),
-      ...(ip ? geo : {}),
-      ...uaInfo,
+      ...(payload.properties ?? {}),
+      country: geo.country,
+      city: geo.city,
+      region: geo.region,
+      longitude: geo.longitude,
+      latitude: geo.latitude,
+      os: uaInfo.os,
+      os_version: uaInfo.osVersion,
+      browser: uaInfo.browser,
+      browser_version: uaInfo.browserVersion,
+      device: uaInfo.device,
+      brand: uaInfo.brand,
+      model: uaInfo.model,
    },
-    ...rest,
  });

-  reply.status(202).send(profileId);
+  reply.status(202).send(payload.profileId);
 }

 export async function incrementProfileProperty(
@@ -65,18 +62,6 @@ export async function incrementProfileProperty(
    return reply.status(400).send('No projectId');
  }

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  const profile = await getProfileById(profileId, projectId);
  if (!profile) {
    return reply.status(404).send('Not found');
@@ -119,18 +104,6 @@ export async function decrementProfileProperty(
    return reply.status(400).send('No projectId');
  }

-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
  const profile = await getProfileById(profileId, projectId);
  if (!profile) {
    return reply.status(404).send('Not found');
--- a/apps/api/src/controllers/track.controller.ts
+++ b/apps/api/src/controllers/track.controller.ts
@@ -1,12 +1,14 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
-import { path, assocPath, pathOr, pick } from 'ramda';
+import { assocPath, pathOr, pick } from 'ramda';

-import { checkDuplicatedEvent } from '@/utils/deduplicate';
+import { logger } from '@/utils/logger';
 import { generateId } from '@openpanel/common';
 import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
 import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
-import { eventsGroupQueue } from '@openpanel/queue';
+import type { ILogger } from '@openpanel/logger';
+import { getEventsGroupQueueShard } from '@openpanel/queue';
+import { getRedisCache } from '@openpanel/redis';
 import type {
  DecrementPayload,
  IdentifyPayload,
@@ -37,10 +39,10 @@ export function getStringHeaders(headers: FastifyRequest['headers']) {
 }

 function getIdentity(body: TrackHandlerPayload): IdentifyPayload | undefined {
-  const identity = path<IdentifyPayload>(
-    ['properties', '__identify'],
-    body.payload,
-  );
+  const identity =
+    'properties' in body.payload
+      ? (body.payload?.properties?.__identify as IdentifyPayload | undefined)
+      : undefined;

  return (
    identity ||
@@ -56,27 +58,28 @@ export function getTimestamp(
  timestamp: FastifyRequest['timestamp'],
  payload: TrackHandlerPayload['payload'],
 ) {
-  const safeTimestamp = new Date(timestamp || Date.now()).toISOString();
-  const userDefinedTimestamp = path<string>(
-    ['properties', '__timestamp'],
-    payload,
-  );
+  const safeTimestamp = timestamp || Date.now();
+  const userDefinedTimestamp =
+    'properties' in payload
+      ? (payload?.properties?.__timestamp as string | undefined)
+      : undefined;

  if (!userDefinedTimestamp) {
    return { timestamp: safeTimestamp, isTimestampFromThePast: false };
  }

  const clientTimestamp = new Date(userDefinedTimestamp);
+  const clientTimestampNumber = clientTimestamp.getTime();

  if (
-    Number.isNaN(clientTimestamp.getTime()) ||
-    clientTimestamp > new Date(safeTimestamp)
+    Number.isNaN(clientTimestampNumber) ||
+    clientTimestampNumber > safeTimestamp
  ) {
    return { timestamp: safeTimestamp, isTimestampFromThePast: false };
  }

  return {
-    timestamp: clientTimestamp.toISOString(),
+    timestamp: clientTimestampNumber,
    isTimestampFromThePast: true,
  };
 }
@@ -89,18 +92,19 @@ export async function handler(
 ) {
  const timestamp = getTimestamp(request.timestamp, request.body.payload);
  const ip =
-    path<string>(['properties', '__ip'], request.body.payload) ||
-    request.clientIp;
+    'properties' in request.body.payload &&
+    request.body.payload.properties?.__ip
+      ? (request.body.payload.properties.__ip as string)
+      : request.clientIp;
  const ua = request.headers['user-agent']!;
  const projectId = request.client?.projectId;

  if (!projectId) {
-    reply.status(400).send({
+    return reply.status(400).send({
      status: 400,
      error: 'Bad Request',
      message: 'Missing projectId',
    });
-    return;
  }

  const identity = getIdentity(request.body);
@@ -132,33 +136,7 @@ export async function handler(
          })
        : '';

-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-            previousDeviceId,
-            currentDeviceId,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
-      const promises = [
-        track({
-          payload: request.body.payload,
-          currentDeviceId,
-          previousDeviceId,
-          projectId,
-          geo,
-          headers: getStringHeaders(request.headers),
-          timestamp: timestamp.timestamp,
-          isTimestampFromThePast: timestamp.isTimestampFromThePast,
-        }),
-      ];
+      const promises = [];

      // If we have more than one property in the identity object, we should identify the user
      // Otherwise its only a profileId and we should not identify the user
@@ -173,23 +151,24 @@ export async function handler(
        );
      }

+      promises.push(
+        track({
+          log: request.log.info,
+          payload: request.body.payload,
+          currentDeviceId,
+          previousDeviceId,
+          projectId,
+          geo,
+          headers: getStringHeaders(request.headers),
+          timestamp: timestamp.timestamp,
+          isTimestampFromThePast: timestamp.isTimestampFromThePast,
+        }),
+      );
+
      await Promise.all(promises);
      break;
    }
    case 'identify': {
-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
      const geo = await getGeoLocation(ip);
      await identify({
        payload: request.body.payload,
@@ -200,27 +179,13 @@ export async function handler(
      break;
    }
    case 'alias': {
-      reply.status(400).send({
+      return reply.status(400).send({
        status: 400,
        error: 'Bad Request',
        message: 'Alias is not supported',
      });
-      break;
    }
    case 'increment': {
-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
      await increment({
        payload: request.body.payload,
        projectId,
@@ -228,19 +193,6 @@ export async function handler(
      break;
    }
    case 'decrement': {
-      if (
-        await checkDuplicatedEvent({
-          reply,
-          payload: {
-            ...request.body,
-            timestamp,
-          },
-          projectId,
-        })
-      ) {
-        return;
-      }
-
      await decrement({
        payload: request.body.payload,
        projectId,
@@ -248,12 +200,11 @@ export async function handler(
      break;
    }
    default: {
-      reply.status(400).send({
+      return reply.status(400).send({
        status: 400,
        error: 'Bad Request',
        message: 'Invalid type',
      });
-      break;
    }
  }

@@ -269,6 +220,7 @@ async function track({
  headers,
  timestamp,
  isTimestampFromThePast,
+  log,
 }: {
  payload: TrackPayload;
  currentDeviceId: string;
@@ -276,8 +228,9 @@ async function track({
  projectId: string;
  geo: GeoLocation;
  headers: Record<string, string | undefined>;
-  timestamp: string;
+  timestamp: number;
  isTimestampFromThePast: boolean;
+  log: any;
 }) {
  const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
  const groupId = uaInfo.isServer
@@ -285,8 +238,14 @@ async function track({
      ? `${projectId}:${payload.profileId}`
      : `${projectId}:${generateId()}`
    : currentDeviceId;
-  await eventsGroupQueue.add({
-    orderMs: new Date(timestamp).getTime(),
+  const jobId = [payload.name, timestamp, projectId, currentDeviceId, groupId]
+    .filter(Boolean)
+    .join('-');
+  await getRedisCache().incr('track:counter');
+  log('track handler', {
+    jobId: jobId,
+    groupId: groupId,
+    timestamp: timestamp,
    data: {
      projectId,
      headers,
@@ -295,11 +254,29 @@ async function track({
        timestamp,
        isTimestampFromThePast,
      },
+      uaInfo,
+      geo,
+      currentDeviceId,
+      previousDeviceId,
+    },
+  });
+  await getEventsGroupQueueShard(groupId).add({
+    orderMs: timestamp,
+    data: {
+      projectId,
+      headers,
+      event: {
+        ...payload,
+        timestamp,
+        isTimestampFromThePast,
+      },
+      uaInfo,
      geo,
      currentDeviceId,
      previousDeviceId,
    },
    groupId,
+    jobId,
  });
 }

@@ -322,8 +299,18 @@ async function identify({
    projectId,
    properties: {
      ...(payload.properties ?? {}),
-      ...(geo ?? {}),
-      ...uaInfo,
+      country: geo.country,
+      city: geo.city,
+      region: geo.region,
+      longitude: geo.longitude,
+      latitude: geo.latitude,
+      os: uaInfo.os,
+      os_version: uaInfo.osVersion,
+      browser: uaInfo.browser,
+      browser_version: uaInfo.browserVersion,
+      device: uaInfo.device,
+      brand: uaInfo.brand,
+      model: uaInfo.model,
    },
  });
 }
--- a/apps/api/src/hooks/duplicate.hook.ts
+++ b/apps/api/src/hooks/duplicate.hook.ts
@@ -0,0 +1,21 @@
+import { isDuplicatedEvent } from '@/utils/deduplicate';
+import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+
+export async function duplicateHook(
+  req: FastifyRequest<{
+    Body: PostEventPayload | TrackHandlerPayload;
+  }>,
+  reply: FastifyReply,
+) {
+  const isDuplicate = await isDuplicatedEvent({
+    ip: req.clientIp ?? '',
+    origin: req.headers.origin ?? '',
+    payload: req.body,
+    projectId: (req.headers['openpanel-client-id'] as string) || '',
+  });
+
+  if (isDuplicate) {
+    return reply.status(200).send('Duplicate event');
+  }
+}
--- a/apps/api/src/hooks/fix.hook.ts
+++ b/apps/api/src/hooks/fix.hook.ts
@@ -1,16 +0,0 @@
-import type { FastifyRequest } from 'fastify';
-
-export async function fixHook(request: FastifyRequest) {
-  const ua = request.headers['user-agent'];
-  // Swift SDK issue: https://github.com/Openpanel-dev/swift-sdk/commit/d588fa761a36a33f3b78eb79d83bfd524e3c7144
-  if (ua) {
-    const regex = /OpenPanel\/(\d+\.\d+\.\d+)\sOpenPanel\/(\d+\.\d+\.\d+)/;
-    const match = ua.match(regex);
-    if (match) {
-      request.headers['user-agent'] = ua.replace(
-        regex,
-        `OpenPanel/${match[1]}`,
-      );
-    }
-  }
-}
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -28,7 +28,6 @@ import {
  liveness,
  readiness,
 } from './controllers/healthcheck.controller';
-import { fixHook } from './hooks/fix.hook';
 import { ipHook } from './hooks/ip.hook';
 import { requestIdHook } from './hooks/request-id.hook';
 import { requestLoggingHook } from './hooks/request-logging.hook';
@@ -125,7 +124,6 @@ const startServer = async () => {
    fastify.addHook('onRequest', requestIdHook);
    fastify.addHook('onRequest', timestampHook);
    fastify.addHook('onRequest', ipHook);
-    fastify.addHook('onRequest', fixHook);
    fastify.addHook('onResponse', requestLoggingHook);

    fastify.register(compress, {
--- a/apps/api/src/routes/event.router.ts
+++ b/apps/api/src/routes/event.router.ts
@@ -2,9 +2,11 @@ import * as controller from '@/controllers/event.controller';
 import type { FastifyPluginCallback } from 'fastify';

 import { clientHook } from '@/hooks/client.hook';
+import { duplicateHook } from '@/hooks/duplicate.hook';
 import { isBotHook } from '@/hooks/is-bot.hook';

 const eventRouter: FastifyPluginCallback = async (fastify) => {
+  fastify.addHook('preValidation', duplicateHook);
  fastify.addHook('preHandler', clientHook);
  fastify.addHook('preHandler', isBotHook);

--- a/apps/api/src/utils/auth.ts
+++ b/apps/api/src/utils/auth.ts
@@ -3,6 +3,7 @@ import type { FastifyRequest, RawRequestDefaultExpression } from 'fastify';
 import { verifyPassword } from '@openpanel/common/server';
 import type { IServiceClientWithProject } from '@openpanel/db';
 import { ClientType, getClientByIdCached } from '@openpanel/db';
+import { getCache } from '@openpanel/redis';
 import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
 import type {
  IProjectFilterIp,
@@ -135,7 +136,13 @@ export async function validateSdkRequest(
  }

  if (client.secret && clientSecret) {
-    if (await verifyPassword(clientSecret, client.secret)) {
+    const isVerified = await getCache(
+      `client:auth:${clientId}:${clientSecret.slice(0, 5)}`,
+      60 * 5,
+      async () => await verifyPassword(clientSecret, client.secret!),
+      true,
+    );
+    if (isVerified) {
      return client;
    }
  }
--- a/apps/api/src/utils/deduplicate.ts
+++ b/apps/api/src/utils/deduplicate.ts
@@ -1,11 +1,14 @@
 import { getLock } from '@openpanel/redis';
 import fastJsonStableHash from 'fast-json-stable-hash';
-import type { FastifyReply } from 'fastify';

 export async function isDuplicatedEvent({
+  ip,
+  origin,
  payload,
  projectId,
 }: {
+  ip: string;
+  origin: string;
  payload: Record<string, any>;
  projectId: string;
 }) {
@@ -13,6 +16,8 @@ export async function isDuplicatedEvent({
    `fastify:deduplicate:${fastJsonStableHash.hash(
      {
        ...payload,
+        ip,
+        origin,
        projectId,
      },
      'md5',
@@ -27,24 +32,3 @@ export async function isDuplicatedEvent({

  return true;
 }
-
-export async function checkDuplicatedEvent({
-  reply,
-  payload,
-  projectId,
-}: {
-  reply: FastifyReply;
-  payload: Record<string, any>;
-  projectId: string;
-}) {
-  if (await isDuplicatedEvent({ payload, projectId })) {
-    reply.log.info('duplicated event', {
-      payload,
-      projectId,
-    });
-    reply.status(200).send('duplicated');
-    return true;
-  }
-
-  return false;
-}
--- a/apps/api/src/utils/graceful-shutdown.ts
+++ b/apps/api/src/utils/graceful-shutdown.ts
@@ -1,7 +1,7 @@
 import { ch, db } from '@openpanel/db';
 import {
  cronQueue,
-  eventsGroupQueue,
+  eventsGroupQueues,
  miscQueue,
  notificationQueue,
  sessionsQueue,
@@ -71,7 +71,7 @@ export async function shutdown(
  // Step 6: Close Bull queues (graceful shutdown of queue state)
  try {
    await Promise.all([
-      eventsGroupQueue.close(),
+      ...eventsGroupQueues.map((queue) => queue.close()),
      sessionsQueue.close(),
      cronQueue.close(),
      miscQueue.close(),
--- a/apps/start/package.json
+++ b/apps/start/package.json
@@ -19,7 +19,6 @@
  },
  "dependencies": {
    "@ai-sdk/react": "^1.2.5",
-    "@clickhouse/client": "^1.2.0",
    "@dnd-kit/core": "^6.3.1",
    "@dnd-kit/sortable": "^10.0.0",
    "@dnd-kit/utilities": "^3.2.2",
--- a/apps/start/src/components/realtime/realtime-geo.tsx
+++ b/apps/start/src/components/realtime/realtime-geo.tsx
@@ -75,7 +75,7 @@ export function RealtimeGeo({ projectId }: RealtimeGeoProps) {
          },
          {
            name: 'Events',
-            width: '84px',
+            width: '60px',
            render(item) {
              return (
                <div className="row gap-2 justify-end">
@@ -86,6 +86,19 @@ export function RealtimeGeo({ projectId }: RealtimeGeoProps) {
              );
            },
          },
+          {
+            name: 'Sessions',
+            width: '82px',
+            render(item) {
+              return (
+                <div className="row gap-2 justify-end">
+                  <span className="font-semibold">
+                    {number.short(item.unique_sessions)}
+                  </span>
+                </div>
+              );
+            },
+          },
        ]}
      />
    </div>
--- a/apps/start/src/components/realtime/realtime-paths.tsx
+++ b/apps/start/src/components/realtime/realtime-paths.tsx
@@ -82,7 +82,7 @@ export function RealtimePaths({ projectId }: RealtimePathsProps) {
          },
          {
            name: 'Events',
-            width: '84px',
+            width: '60px',
            render(item) {
              return (
                <div className="row gap-2 justify-end">
@@ -93,6 +93,19 @@ export function RealtimePaths({ projectId }: RealtimePathsProps) {
              );
            },
          },
+          {
+            name: 'Sessions',
+            width: '82px',
+            render(item) {
+              return (
+                <div className="row gap-2 justify-end">
+                  <span className="font-semibold">
+                    {number.short(item.unique_sessions)}
+                  </span>
+                </div>
+              );
+            },
+          },
        ]}
      />
    </div>
--- a/apps/start/src/components/realtime/realtime-referrals.tsx
+++ b/apps/start/src/components/realtime/realtime-referrals.tsx
@@ -65,7 +65,7 @@ export function RealtimeReferrals({ projectId }: RealtimeReferralsProps) {
          },
          {
            name: 'Events',
-            width: '84px',
+            width: '60px',
            render(item) {
              return (
                <div className="row gap-2 justify-end">
@@ -76,6 +76,19 @@ export function RealtimeReferrals({ projectId }: RealtimeReferralsProps) {
              );
            },
          },
+          {
+            name: 'Sessions',
+            width: '82px',
+            render(item) {
+              return (
+                <div className="row gap-2 justify-end">
+                  <span className="font-semibold">
+                    {number.short(item.unique_sessions)}
+                  </span>
+                </div>
+              );
+            },
+          },
        ]}
      />
    </div>
--- a/apps/start/src/components/ui/data-table/data-table-hooks.tsx
+++ b/apps/start/src/components/ui/data-table/data-table-hooks.tsx
@@ -4,8 +4,8 @@ import type {
  VisibilityState,
 } from '@tanstack/react-table';
 import { parseAsInteger, useQueryState } from 'nuqs';
-import { useState } from 'react';
-import { useLocalStorage } from 'usehooks-ts';
+import { useEffect, useState } from 'react';
+import { useLocalStorage, useReadLocalStorage } from 'usehooks-ts';

 export const useDataTablePagination = (pageSize = 10) => {
  const [page, setPage] = useQueryState(
@@ -22,6 +22,12 @@ export const useDataTablePagination = (pageSize = 10) => {
  return { page, setPage, state };
 };

+export const useReadColumnVisibility = (persistentKey: string) => {
+  return useReadLocalStorage<Record<string, boolean>>(
+    `@op:${persistentKey}-column-visibility`,
+  );
+};
+
 export const useDataTableColumnVisibility = <TData,>(
  columns: ColumnDef<TData>[],
  persistentKey: string,
@@ -43,6 +49,13 @@ export const useDataTableColumnVisibility = <TData,>(
    }, {} as VisibilityState),
  );

+  // somewhat hack
+  // Set initial column visibility,
+  // otherwise will not useReadColumnVisibility be updated
+  useEffect(() => {
+    setColumnVisibility(columnVisibility);
+  }, []);
+
  const [columnOrder, setColumnOrder] = useLocalStorage<string[]>(
    `@op:${persistentKey}-column-order`,
    columns.map((column) => column.id!),
--- a/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.conversions.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.conversions.tsx
@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import { useEventQueryNamesFilter } from '@/hooks/use-event-query-filters';
 import { useTRPC } from '@/integrations/trpc/react';
 import { useInfiniteQuery } from '@tanstack/react-query';
@@ -20,6 +21,7 @@ function Component() {
  );
  const [endDate, setEndDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
  const query = useInfiniteQuery(
    trpc.event.conversions.infiniteQueryOptions(
      {
@@ -27,6 +29,7 @@ function Component() {
        startDate: startDate || undefined,
        endDate: endDate || undefined,
        events: eventNames,
+        columnVisibility: columnVisibility ?? {},
      },
      {
        getNextPageParam: (lastPage) => lastPage.meta.next,
--- a/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.events._tabs.events.tsx
@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
  useEventQueryFilters,
  useEventQueryNamesFilter,
@@ -21,6 +22,8 @@ function Component() {
  const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
  const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
+
  const query = useInfiniteQuery(
    trpc.event.events.infiniteQueryOptions(
      {
@@ -30,8 +33,10 @@ function Component() {
        profileId: '',
        startDate: startDate || undefined,
        endDate: endDate || undefined,
+        columnVisibility: columnVisibility ?? {},
      },
      {
+        enabled: columnVisibility !== null,
        getNextPageParam: (lastPage) => lastPage.meta.next,
      },
    ),
--- a/apps/start/src/routes/_app.$organizationId.$projectId.profiles.$profileId._tabs.events.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.profiles.$profileId._tabs.events.tsx
@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
  useEventQueryFilters,
  useEventQueryNamesFilter,
@@ -21,6 +22,7 @@ function Component() {
  const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
  const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
  const query = useInfiniteQuery(
    trpc.event.events.infiniteQueryOptions(
      {
@@ -30,8 +32,10 @@ function Component() {
        startDate: startDate || undefined,
        endDate: endDate || undefined,
        events: eventNames,
+        columnVisibility: columnVisibility ?? {},
      },
      {
+        enabled: columnVisibility !== null,
        getNextPageParam: (lastPage) => lastPage.meta.next,
      },
    ),
--- a/apps/start/src/routes/_app.$organizationId.$projectId.sessions_.$sessionId.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId.sessions_.$sessionId.tsx
@@ -3,12 +3,11 @@ import FullPageLoadingState from '@/components/full-page-loading-state';
 import { PageContainer } from '@/components/page-container';
 import { PageHeader } from '@/components/page-header';
 import { SerieIcon } from '@/components/report-chart/common/serie-icon';
-import { useDataTablePagination } from '@/components/ui/data-table/data-table-hooks';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
  useEventQueryFilters,
  useEventQueryNamesFilter,
 } from '@/hooks/use-event-query-filters';
-import { useSearchQueryState } from '@/hooks/use-search-query-state';
 import { useTRPC } from '@/integrations/trpc/react';
 import { createProjectTitle } from '@/utils/title';
 import { useInfiniteQuery, useSuspenseQuery } from '@tanstack/react-query';
@@ -46,8 +45,6 @@ function Component() {
  const trpc = useTRPC();

  const LIMIT = 50;
-  const { page } = useDataTablePagination(LIMIT);
-  const { debouncedSearch } = useSearchQueryState();

  const { data: session } = useSuspenseQuery(
    trpc.session.byId.queryOptions({
@@ -60,7 +57,7 @@ function Component() {
  const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
  const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
  const [eventNames] = useEventQueryNamesFilter();
-
+  const columnVisibility = useReadColumnVisibility('events');
  const query = useInfiniteQuery(
    trpc.event.events.infiniteQueryOptions(
      {
@@ -70,8 +67,10 @@ function Component() {
        events: eventNames,
        startDate: startDate || undefined,
        endDate: endDate || undefined,
+        columnVisibility: columnVisibility ?? {},
      },
      {
+        enabled: columnVisibility !== null,
        getNextPageParam: (lastPage) => lastPage.meta.next,
      },
    ),
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -24,7 +24,7 @@
    "@openpanel/redis": "workspace:*",
    "bullmq": "^5.8.7",
    "express": "^4.18.2",
-    "groupmq": "1.0.0-next.19",
+    "groupmq": "1.1.0-next.5",
    "prom-client": "^15.1.3",
    "ramda": "^0.29.1",
    "source-map-support": "^0.5.21",
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -2,75 +2,221 @@ import type { Queue, WorkerOptions } from 'bullmq';
 import { Worker } from 'bullmq';

 import {
+  EVENTS_GROUP_QUEUES_SHARDS,
  type EventsQueuePayloadIncomingEvent,
  cronQueue,
-  eventsGroupQueue,
+  eventsGroupQueues,
  importQueue,
  miscQueue,
  notificationQueue,
  queueLogger,
  sessionsQueue,
 } from '@openpanel/queue';
-import { getRedisQueue } from '@openpanel/redis';
+import { getLock, getRedisQueue } from '@openpanel/redis';

 import { performance } from 'node:perf_hooks';
 import { setTimeout as sleep } from 'node:timers/promises';
 import { Worker as GroupWorker } from 'groupmq';

 import { cronJob } from './jobs/cron';
-import { eventsJob } from './jobs/events';
 import { incomingEventPure } from './jobs/events.incoming-event';
 import { importJob } from './jobs/import';
 import { miscJob } from './jobs/misc';
 import { notificationJob } from './jobs/notification';
 import { sessionsJob } from './jobs/sessions';
+import { eventsGroupJobDuration } from './metrics';
 import { logger } from './utils/logger';
+import { requireSingleton } from './utils/singleton-lock';

 const workerOptions: WorkerOptions = {
  connection: getRedisQueue(),
 };

-export async function bootWorkers() {
-  const eventsGroupWorker = new GroupWorker<
-    EventsQueuePayloadIncomingEvent['payload']
-  >({
-    concurrency: Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '1', 10),
-    logger: queueLogger,
-    queue: eventsGroupQueue,
-    handler: async (job) => {
-      logger.info('processing event (group queue)', {
-        groupId: job.groupId,
-        timestamp: job.data.event.timestamp,
-      });
-      await incomingEventPure(job.data);
-    },
-  });
-  eventsGroupWorker.run();
-  const sessionsWorker = new Worker(
-    sessionsQueue.name,
-    sessionsJob,
-    workerOptions,
-  );
-  const cronWorker = new Worker(cronQueue.name, cronJob, workerOptions);
-  const notificationWorker = new Worker(
-    notificationQueue.name,
-    notificationJob,
-    workerOptions,
-  );
-  const miscWorker = new Worker(miscQueue.name, miscJob, workerOptions);
-  const importWorker = new Worker(importQueue.name, importJob, {
-    ...workerOptions,
-    concurrency: Number.parseInt(process.env.IMPORT_JOB_CONCURRENCY || '1', 10),
-  });
+type QueueName = string; // Can be: events, events_N (where N is 0 to shards-1), sessions, cron, notification, misc

-  const workers = [
-    sessionsWorker,
-    cronWorker,
-    notificationWorker,
-    miscWorker,
-    importWorker,
-    // eventsGroupWorker,
-  ];
+/**
+ * Parses the ENABLED_QUEUES environment variable and returns an array of queue names to start.
+ * If no env var is provided, returns all queues.
+ *
+ * Supported queue names:
+ * - events - All event shards (events_0, events_1, ..., events_N)
+ * - events_N - Individual event shard (where N is 0 to EVENTS_GROUP_QUEUES_SHARDS-1)
+ * - sessions, cron, notification, misc
+ */
+function getEnabledQueues(): QueueName[] {
+  const enabledQueuesEnv = process.env.ENABLED_QUEUES?.trim();
+
+  if (!enabledQueuesEnv) {
+    logger.info('No ENABLED_QUEUES specified, starting all queues', {
+      totalEventShards: EVENTS_GROUP_QUEUES_SHARDS,
+    });
+    return ['events', 'sessions', 'cron', 'notification', 'misc', 'import'];
+  }
+
+  const queues = enabledQueuesEnv
+    .split(',')
+    .map((q) => q.trim())
+    .filter(Boolean);
+
+  logger.info('Starting queues from ENABLED_QUEUES', {
+    queues,
+    totalEventShards: EVENTS_GROUP_QUEUES_SHARDS,
+  });
+  return queues;
+}
+
+/**
+ * Gets the concurrency setting for a queue from environment variables.
+ * Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., EVENTS_0_CONCURRENCY=32)
+ */
+function getConcurrencyFor(queueName: string, defaultValue = 1): number {
+  const envKey = `${queueName.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_CONCURRENCY`;
+  const value = process.env[envKey];
+
+  if (value) {
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isNaN(parsed) && parsed > 0) {
+      return parsed;
+    }
+  }
+
+  return defaultValue;
+}
+
+export async function bootWorkers() {
+  const enabledQueues = getEnabledQueues();
+  const enforceSingleton = process.env.ENFORCE_SINGLETON === '1';
+  let singletonCleanup: (() => void) | null = null;
+
+  // Enforce singleton lock if requested
+  if (enforceSingleton) {
+    const lockKey = enabledQueues.join(',');
+    logger.info('Enforcing singleton mode', { lockKey });
+    singletonCleanup = await requireSingleton(lockKey);
+  }
+
+  const workers: (Worker | GroupWorker<any>)[] = [];
+
+  // Start event workers based on enabled queues
+  const eventQueuesToStart: number[] = [];
+
+  if (enabledQueues.includes('events')) {
+    // Start all event shards
+    for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
+      eventQueuesToStart.push(i);
+    }
+  } else {
+    // Start specific event shards (events_0, events_1, etc.)
+    for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
+      if (enabledQueues.includes(`events_${i}`)) {
+        eventQueuesToStart.push(i);
+      }
+    }
+  }
+
+  for (const index of eventQueuesToStart) {
+    const queue = eventsGroupQueues[index];
+    if (!queue) continue;
+
+    const queueName = `events_${index}`;
+    const concurrency = getConcurrencyFor(
+      queueName,
+      Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10),
+    );
+
+    const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
+      queue,
+      concurrency,
+      logger: queueLogger,
+      blockingTimeoutSec: Number.parseFloat(
+        process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',
+      ),
+      handler: async (job) => {
+        if (await getLock(job.id, '1', 10000)) {
+          logger.info('worker handler', {
+            jobId: job.id,
+            groupId: job.groupId,
+            timestamp: job.data.event.timestamp,
+            data: job.data,
+          });
+        } else {
+          logger.info('event already processed', {
+            jobId: job.id,
+            groupId: job.groupId,
+            timestamp: job.data.event.timestamp,
+            data: job.data,
+          });
+        }
+        await incomingEventPure(job.data);
+      },
+    });
+
+    worker.run();
+    workers.push(worker);
+    logger.info(`Started worker for ${queueName}`, { concurrency });
+  }
+
+  // Start sessions worker
+  if (enabledQueues.includes('sessions')) {
+    const concurrency = getConcurrencyFor('sessions');
+    const sessionsWorker = new Worker(sessionsQueue.name, sessionsJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(sessionsWorker);
+    logger.info('Started worker for sessions', { concurrency });
+  }
+
+  // Start cron worker
+  if (enabledQueues.includes('cron')) {
+    const concurrency = getConcurrencyFor('cron');
+    const cronWorker = new Worker(cronQueue.name, cronJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(cronWorker);
+    logger.info('Started worker for cron', { concurrency });
+  }
+
+  // Start notification worker
+  if (enabledQueues.includes('notification')) {
+    const concurrency = getConcurrencyFor('notification');
+    const notificationWorker = new Worker(
+      notificationQueue.name,
+      notificationJob,
+      { ...workerOptions, concurrency },
+    );
+    workers.push(notificationWorker);
+    logger.info('Started worker for notification', { concurrency });
+  }
+
+  // Start misc worker
+  if (enabledQueues.includes('misc')) {
+    const concurrency = getConcurrencyFor('misc');
+    const miscWorker = new Worker(miscQueue.name, miscJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(miscWorker);
+    logger.info('Started worker for misc', { concurrency });
+  }
+
+  // Start import worker
+  if (enabledQueues.includes('import')) {
+    const concurrency = getConcurrencyFor('import');
+    const importWorker = new Worker(importQueue.name, importJob, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(importWorker);
+    logger.info('Started worker for misc', { concurrency });
+  }
+
+  if (workers.length === 0) {
+    logger.warn(
+      'No workers started. Check ENABLED_QUEUES environment variable.',
+    );
+  }

  workers.forEach((worker) => {
    (worker as Worker).on('error', (error) => {
@@ -94,6 +240,13 @@ export async function bootWorkers() {

    (worker as Worker).on('failed', (job) => {
      if (job) {
+        if (job.processedOn && job.finishedOn) {
+          const duration = job.finishedOn - job.processedOn;
+          eventsGroupJobDuration.observe(
+            { queue_shard: worker.name, status: 'failed' },
+            duration,
+          );
+        }
        logger.error('job failed', {
          jobId: job.id,
          worker: worker.name,
@@ -106,15 +259,13 @@ export async function bootWorkers() {

    (worker as Worker).on('completed', (job) => {
      if (job) {
-        logger.info('job completed', {
-          jobId: job.id,
-          worker: worker.name,
-          data: job.data,
-          elapsed:
-            job.processedOn && job.finishedOn
-              ? job.finishedOn - job.processedOn
-              : undefined,
-        });
+        if (job.processedOn && job.finishedOn) {
+          const duration = job.finishedOn - job.processedOn;
+          eventsGroupJobDuration.observe(
+            { queue_shard: worker.name, status: 'success' },
+            duration,
+          );
+        }
      }
    });

@@ -135,8 +286,19 @@ export async function bootWorkers() {
    });
    try {
      const time = performance.now();
-      await waitForQueueToEmpty(cronQueue);
+
+      // Wait for cron queue to empty if it's running
+      if (enabledQueues.includes('cron')) {
+        await waitForQueueToEmpty(cronQueue);
+      }
+
      await Promise.all(workers.map((worker) => worker.close()));
+
+      // Release singleton lock if acquired
+      if (singletonCleanup) {
+        singletonCleanup();
+      }
+
      logger.info('workers closed successfully', {
        elapsed: performance.now() - time,
      });
@@ -155,15 +317,7 @@ export async function bootWorkers() {
  ['uncaughtException', 'unhandledRejection', 'SIGTERM', 'SIGINT'].forEach(
    (evt) => {
      process.on(evt, (code) => {
-        if (process.env.NODE_ENV === 'production') {
-          exitHandler(evt, code);
-        } else {
-          logger.info('Shutting down for development', {
-            event: evt,
-            code,
-          });
-          process.exit(0);
-        }
+        exitHandler(evt, code);
      });
    },
  );
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -34,7 +34,9 @@ async function start() {
    serverAdapter.setBasePath('/');
    createBullBoard({
      queues: [
-        new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
+        ...eventsGroupQueues.map(
+          (queue) => new BullBoardGroupMQAdapter(queue) as any,
+        ),
        new BullMQAdapter(sessionsQueue),
        new BullMQAdapter(cronQueue),
        new BullMQAdapter(notificationQueue),
--- a/apps/worker/src/jobs/events.create-session-end.ts
+++ b/apps/worker/src/jobs/events.create-session-end.ts
@@ -68,7 +68,7 @@ export async function createSessionEnd(
    reqId: payload.properties?.__reqId ?? 'unknown',
  });

-  logger.info('Processing session end job');
+  logger.debug('Processing session end job');

  const session = await sessionBuffer.getExistingSession(payload.sessionId);

--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -18,9 +18,9 @@ import {
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
-import type { Job } from 'bullmq';
+import { getLock, getRedisCache } from '@openpanel/redis';
+import { DelayedError, type Job } from 'bullmq';
 import * as R from 'ramda';
-import { omit } from 'ramda';
 import { v4 as uuid } from 'uuid';

 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
@@ -56,6 +56,7 @@ export async function incomingEventPure(
  job?: Job<EventsQueuePayloadIncomingEvent>,
  token?: string,
 ) {
+  await getRedisCache().incr('queue:counter');
  const {
    geo,
    event: body,
@@ -63,6 +64,7 @@ export async function incomingEventPure(
    projectId,
    currentDeviceId,
    previousDeviceId,
+    uaInfo: _uaInfo,
  } = jobPayload;
  const properties = body.properties ?? {};
  const reqId = headers['request-id'] ?? 'unknown';
@@ -93,13 +95,14 @@ export async function incomingEventPure(
  const userAgent = headers['user-agent'];
  const sdkName = headers['openpanel-sdk-name'];
  const sdkVersion = headers['openpanel-sdk-version'];
-  const uaInfo = parseUserAgent(userAgent, properties);
+  // TODO: Remove both user-agent and parseUserAgent
+  const uaInfo = _uaInfo ?? parseUserAgent(userAgent, properties);

  const baseEvent = {
    name: body.name,
    profileId,
    projectId,
-    properties: omit(GLOBAL_PROPERTIES, {
+    properties: R.omit(GLOBAL_PROPERTIES, {
      ...properties,
      __user_agent: userAgent,
      __hash: hash,
--- a/apps/worker/src/metrics.ts
+++ b/apps/worker/src/metrics.ts
@@ -2,23 +2,33 @@ import client from 'prom-client';

 import {
  botBuffer,
-  db,
  eventBuffer,
  profileBuffer,
  sessionBuffer,
 } from '@openpanel/db';
-import { cronQueue, eventsGroupQueue, sessionsQueue } from '@openpanel/queue';
+import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';

 const Registry = client.Registry;

 export const register = new Registry();

-const queues = [sessionsQueue, cronQueue, eventsGroupQueue];
+const queues = [sessionsQueue, cronQueue, ...eventsGroupQueues];
+
+// Histogram to track job processing time for eventsGroupQueues
+export const eventsGroupJobDuration = new client.Histogram({
+  name: 'events_group_job_duration_ms',
+  help: 'Duration of job processing in eventsGroupQueues (in ms)',
+  labelNames: ['queue_shard', 'status'],
+  buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
+  registers: [register],
+});
+
+register.registerMetric(eventsGroupJobDuration);

 queues.forEach((queue) => {
  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_active_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
      help: 'Active count',
      async collect() {
        const metric = await queue.getActiveCount();
@@ -29,7 +39,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_delayed_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
      help: 'Delayed count',
      async collect() {
        const metric = await queue.getDelayedCount();
@@ -40,7 +50,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_failed_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
      help: 'Failed count',
      async collect() {
        const metric = await queue.getFailedCount();
@@ -51,7 +61,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_completed_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
      help: 'Completed count',
      async collect() {
        const metric = await queue.getCompletedCount();
@@ -62,7 +72,7 @@ queues.forEach((queue) => {

  register.registerMetric(
    new client.Gauge({
-      name: `${queue.name}_waiting_count`,
+      name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
      help: 'Waiting count',
      async collect() {
        const metric = await queue.getWaitingCount();
--- a/apps/worker/src/utils/session-handler.ts
+++ b/apps/worker/src/utils/session-handler.ts
@@ -113,7 +113,7 @@ export async function getSessionEndJob(args: {
  } | null> {
    const state = await job.getState();
    if (state !== 'delayed') {
-      logger.info(`[session-handler] Session end job is in "${state}" state`, {
+      logger.debug(`[session-handler] Session end job is in "${state}" state`, {
        state,
        retryCount,
        jobTimestamp: new Date(job.timestamp).toISOString(),
--- a/apps/worker/src/utils/singleton-lock.ts
+++ b/apps/worker/src/utils/singleton-lock.ts
@@ -0,0 +1,69 @@
+import { getLock, getRedisCache } from '@openpanel/redis';
+import { logger } from './logger';
+
+/**
+ * Acquires a distributed lock to ensure only one instance of a worker group runs.
+ * If the lock cannot be acquired, the process exits.
+ *
+ * @param key - The lock key (e.g., 'utility-queues')
+ * @param ttlMs - Time to live for the lock in milliseconds (default: 60000)
+ * @returns A cleanup function that releases the lock
+ */
+export async function requireSingleton(
+  key: string,
+  ttlMs = 60_000,
+): Promise<() => void> {
+  const lockKey = `lock:singleton:${key}`;
+  const lockValue = `${process.pid}-${Date.now()}`;
+
+  // Try to acquire the lock
+  const acquired = await getLock(lockKey, lockValue, ttlMs);
+
+  if (!acquired) {
+    logger.error(
+      `Another instance holds singleton lock for "${key}". Exiting.`,
+      { key },
+    );
+    process.exit(0);
+  }
+
+  logger.debug('Acquired singleton lock', { key, ttlMs, lockValue });
+
+  // Set up automatic extension to keep the lock alive
+  const extensionInterval = setInterval(async () => {
+    try {
+      // Extend the lock by setting it again with the same value
+      const redis = getRedisCache();
+      const result = await redis.set(lockKey, lockValue, 'PX', ttlMs, 'XX');
+
+      if (result === 'OK') {
+        logger.debug('Extended singleton lock', { key });
+      } else {
+        // Lock was lost (someone else acquired it or it expired)
+        logger.error('Lost singleton lock - exiting', { key });
+        clearInterval(extensionInterval);
+        process.exit(1);
+      }
+    } catch (error: unknown) {
+      logger.error('Failed to extend singleton lock - exiting', {
+        key,
+        error,
+      });
+      clearInterval(extensionInterval);
+      process.exit(1);
+    }
+  }, ttlMs / 2);
+
+  // Return cleanup function
+  return () => {
+    clearInterval(extensionInterval);
+    getRedisCache()
+      .del(lockKey)
+      .then(() => {
+        logger.debug('Released singleton lock', { key });
+      })
+      .catch((error: unknown) => {
+        logger.error('Failed to release singleton lock', { key, error });
+      });
+  };
+}