fix: overall perf improvements

* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert

packages/queue/src/queues.ts

@@ -1,5 +1,6 @@
 import { Queue, QueueEvents } from 'bullmq';
 
+import { createHash } from 'node:crypto';
 import type {
   IServiceCreateEventPayload,
   IServiceEvent,
@@ -10,6 +11,21 @@ import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
 import type { TrackPayload } from '@openpanel/sdk';
 import { Queue as GroupQueue } from 'groupmq';
 
+export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(
+  process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',
+  10,
+);
+
+export const getQueueName = (name: string) =>
+  process.env.QUEUE_CLUSTER ? `{${name}}` : name;
+
+function pickShard(projectId: string) {
+  const h = createHash('sha1').update(projectId).digest(); // 20 bytes
+  // take first 4 bytes as unsigned int
+  const x = h.readUInt32BE(0);
+  return x % EVENTS_GROUP_QUEUES_SHARDS; // 0..n-1
+}
+
 export const queueLogger = createLogger({ name: 'queue' });
 
 export interface EventsQueuePayloadIncomingEvent {
@@ -17,9 +33,30 @@ export interface EventsQueuePayloadIncomingEvent {
   payload: {
     projectId: string;
     event: TrackPayload & {
-      timestamp: string;
+      timestamp: string | number;
       isTimestampFromThePast: boolean;
     };
+    uaInfo:
+      | {
+          readonly isServer: true;
+          readonly device: 'server';
+          readonly os: '';
+          readonly osVersion: '';
+          readonly browser: '';
+          readonly browserVersion: '';
+          readonly brand: '';
+          readonly model: '';
+        }
+      | {
+          readonly os: string | undefined;
+          readonly osVersion: string | undefined;
+          readonly browser: string | undefined;
+          readonly browserVersion: string | undefined;
+          readonly device: string;
+          readonly brand: string | undefined;
+          readonly model: string | undefined;
+          readonly isServer: false;
+        };
     geo: {
       country: string | undefined;
       city: string | undefined;
@@ -93,54 +130,70 @@ export type MiscQueuePayload = MiscQueuePayloadTrialEndingSoon;
 
 export type CronQueueType = CronQueuePayload['type'];
 
-const orderingWindowMs = Number.parseInt(
-  process.env.ORDERING_WINDOW_MS || '50',
-  10,
-);
-const orderingGracePeriodDecay = Number.parseFloat(
-  process.env.ORDERING_GRACE_PERIOD_DECAY || '0.9',
-);
-const orderingMaxWaitMultiplier = Number.parseInt(
-  process.env.ORDERING_MAX_WAIT_MULTIPLIER || '8',
+const orderingDelayMs = Number.parseInt(
+  process.env.ORDERING_DELAY_MS || '100',
   10,
 );
 
-export const eventsGroupQueue = new GroupQueue<
-  EventsQueuePayloadIncomingEvent['payload']
->({
-  logger: queueLogger,
-  namespace: 'group_events',
-  redis: getRedisGroupQueue(),
-  orderingMethod: 'in-memory',
-  orderingWindowMs,
-  orderingGracePeriodDecay,
-  orderingMaxWaitMultiplier,
-  keepCompleted: 10,
-  keepFailed: 10_000,
-});
+const autoBatchMaxWaitMs = Number.parseInt(
+  process.env.AUTO_BATCH_MAX_WAIT_MS || '0',
+  10,
+);
+const autoBatchSize = Number.parseInt(process.env.AUTO_BATCH_SIZE || '0', 10);
 
-export const sessionsQueue = new Queue<SessionsQueuePayload>('sessions', {
-  // @ts-ignore
-  connection: getRedisQueue(),
-  defaultJobOptions: {
-    removeOnComplete: 10,
+export const eventsGroupQueues = Array.from({
+  length: EVENTS_GROUP_QUEUES_SHARDS,
+}).map(
+  (_, index, list) =>
+    new GroupQueue<EventsQueuePayloadIncomingEvent['payload']>({
+      logger: queueLogger,
+      namespace: getQueueName(
+        list.length === 1 ? 'group_events' : `group_events_${index}`,
+      ),
+      redis: getRedisGroupQueue(),
+      keepCompleted: 1_000,
+      keepFailed: 10_000,
+      orderingDelayMs: orderingDelayMs,
+      autoBatch:
+        autoBatchMaxWaitMs && autoBatchSize
+          ? {
+              maxWaitMs: autoBatchMaxWaitMs,
+              size: autoBatchSize,
+            }
+          : undefined,
+    }),
+);
+
+export const getEventsGroupQueueShard = (groupId: string) => {
+  const shard = pickShard(groupId);
+  const queue = eventsGroupQueues[shard];
+  if (!queue) {
+    throw new Error(`Queue not found for group ${groupId}`);
+  }
+  return queue;
+};
+
+export const sessionsQueue = new Queue<SessionsQueuePayload>(
+  getQueueName('sessions'),
+  {
+    connection: getRedisQueue(),
+    defaultJobOptions: {
+      removeOnComplete: 10,
+    },
   },
-});
-export const sessionsQueueEvents = new QueueEvents('sessions', {
-  // @ts-ignore
+);
+export const sessionsQueueEvents = new QueueEvents(getQueueName('sessions'), {
   connection: getRedisQueue(),
 });
 
-export const cronQueue = new Queue<CronQueuePayload>('cron', {
-  // @ts-ignore
+export const cronQueue = new Queue<CronQueuePayload>(getQueueName('cron'), {
   connection: getRedisQueue(),
   defaultJobOptions: {
     removeOnComplete: 10,
   },
 });
 
-export const miscQueue = new Queue<MiscQueuePayload>('misc', {
-  // @ts-ignore
+export const miscQueue = new Queue<MiscQueuePayload>(getQueueName('misc'), {
   connection: getRedisQueue(),
   defaultJobOptions: {
     removeOnComplete: 10,
@@ -155,9 +208,8 @@ export type NotificationQueuePayload = {
 };
 
 export const notificationQueue = new Queue<NotificationQueuePayload>(
-  'notification',
+  getQueueName('notification'),
   {
-    // @ts-ignore
     connection: getRedisQueue(),
     defaultJobOptions: {
       removeOnComplete: 10,
@@ -172,13 +224,16 @@ export type ImportQueuePayload = {
   };
 };
 
-export const importQueue = new Queue<ImportQueuePayload>('import', {
-  connection: getRedisQueue(),
-  defaultJobOptions: {
-    removeOnComplete: 10,
-    removeOnFail: 50,
+export const importQueue = new Queue<ImportQueuePayload>(
+  getQueueName('import'),
+  {
+    connection: getRedisQueue(),
+    defaultJobOptions: {
+      removeOnComplete: 10,
+      removeOnFail: 50,
+    },
   },
-});
+);
 
 export function addTrialEndingSoonJob(organizationId: string, delay: number) {
   return miscQueue.add(