fix: overall perf improvements

* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert

apps/api/src/bots/index.ts

@@ -1,19 +1,47 @@
+import { cacheable, cacheableLru } from '@openpanel/redis';
 import bots from './bots';
 
-export function isBot(ua: string) {
-  const res = bots.find((bot) => {
-    if (new RegExp(bot.regex).test(ua)) {
-      return true;
-    }
-    return false;
-  });
-
-  if (!res) {
-    return null;
+// Pre-compile regex patterns at module load time
+const compiledBots = bots.map((bot) => {
+  if ('regex' in bot) {
+    return {
+      ...bot,
+      compiledRegex: new RegExp(bot.regex),
+    };
   }
+  return bot;
+});
 
-  return {
-    name: res.name,
-    type: 'category' in res ? res.category : 'Unknown',
-  };
-}
+const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
+const includesBots = compiledBots.filter((bot) => 'includes' in bot);
+
+export const isBot = cacheableLru(
+  'is-bot',
+  (ua: string) => {
+    // Check simple string patterns first (fast)
+    for (const bot of includesBots) {
+      if (ua.includes(bot.includes)) {
+        return {
+          name: bot.name,
+          type: 'category' in bot ? bot.category : 'Unknown',
+        };
+      }
+    }
+
+    // Check regex patterns (slower)
+    for (const bot of regexBots) {
+      if (bot.compiledRegex.test(ua)) {
+        return {
+          name: bot.name,
+          type: 'category' in bot ? bot.category : 'Unknown',
+        };
+      }
+    }
+
+    return null;
+  },
+  {
+    maxSize: 1000,
+    ttl: 60 * 5,
+  },
+);