fix: overall perf improvements

* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert

apps/api/src/controllers/profile.controller.ts

@@ -1,7 +1,6 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { assocPath, pathOr } from 'ramda';
 
-import { checkDuplicatedEvent, isDuplicatedEvent } from '@/utils/deduplicate';
 import { parseUserAgent } from '@openpanel/common/server';
 import { getProfileById, upsertProfile } from '@openpanel/db';
 import { getGeoLocation } from '@openpanel/geo';
@@ -16,41 +15,39 @@ export async function updateProfile(
   }>,
   reply: FastifyReply,
 ) {
-  const { profileId, properties, ...rest } = request.body;
+  const payload = request.body;
   const projectId = request.client!.projectId;
   if (!projectId) {
     return reply.status(400).send('No projectId');
   }
   const ip = request.clientIp;
   const ua = request.headers['user-agent']!;
-  const uaInfo = parseUserAgent(ua, properties);
+  const uaInfo = parseUserAgent(ua, payload.properties);
   const geo = await getGeoLocation(ip);
 
-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
   await upsertProfile({
-    id: profileId,
+    ...payload,
+    id: payload.profileId,
     isExternal: true,
     projectId,
     properties: {
-      ...(properties ?? {}),
-      ...(ip ? geo : {}),
-      ...uaInfo,
+      ...(payload.properties ?? {}),
+      country: geo.country,
+      city: geo.city,
+      region: geo.region,
+      longitude: geo.longitude,
+      latitude: geo.latitude,
+      os: uaInfo.os,
+      os_version: uaInfo.osVersion,
+      browser: uaInfo.browser,
+      browser_version: uaInfo.browserVersion,
+      device: uaInfo.device,
+      brand: uaInfo.brand,
+      model: uaInfo.model,
     },
-    ...rest,
   });
 
-  reply.status(202).send(profileId);
+  reply.status(202).send(payload.profileId);
 }
 
 export async function incrementProfileProperty(
@@ -65,18 +62,6 @@ export async function incrementProfileProperty(
     return reply.status(400).send('No projectId');
   }
 
-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
   const profile = await getProfileById(profileId, projectId);
   if (!profile) {
     return reply.status(404).send('Not found');
@@ -119,18 +104,6 @@ export async function decrementProfileProperty(
     return reply.status(400).send('No projectId');
   }
 
-  if (
-    await checkDuplicatedEvent({
-      reply,
-      payload: {
-        ...request.body,
-      },
-      projectId,
-    })
-  ) {
-    return;
-  }
-
   const profile = await getProfileById(profileId, projectId);
   if (!profile) {
     return reply.status(404).send('Not found');