fix: overall perf improvements

* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert

apps/api/src/utils/deduplicate.ts

@@ -1,11 +1,14 @@
 import { getLock } from '@openpanel/redis';
 import fastJsonStableHash from 'fast-json-stable-hash';
-import type { FastifyReply } from 'fastify';
 
 export async function isDuplicatedEvent({
+  ip,
+  origin,
   payload,
   projectId,
 }: {
+  ip: string;
+  origin: string;
   payload: Record<string, any>;
   projectId: string;
 }) {
@@ -13,6 +16,8 @@ export async function isDuplicatedEvent({
     `fastify:deduplicate:${fastJsonStableHash.hash(
       {
         ...payload,
+        ip,
+        origin,
         projectId,
       },
       'md5',
@@ -27,24 +32,3 @@ export async function isDuplicatedEvent({
 
   return true;
 }
-
-export async function checkDuplicatedEvent({
-  reply,
-  payload,
-  projectId,
-}: {
-  reply: FastifyReply;
-  payload: Record<string, any>;
-  projectId: string;
-}) {
-  if (await isDuplicatedEvent({ payload, projectId })) {
-    reply.log.info('duplicated event', {
-      payload,
-      projectId,
-    });
-    reply.status(200).send('duplicated');
-    return true;
-  }
-
-  return false;
-}