fix: overall perf improvements

* fix: ignore private ips * fix: performance related fixes * fix: simply event buffer * fix: default to 1 events queue shard * add: cleanup scripts * fix: comments * fix comments * fix * fix: groupmq * wip * fix: sync cachable * remove cluster names and add it behind env flag (if someone want to scale) * fix * wip * better logger * remove reqid and user agent * fix lock * remove wait_for_async_insert
2025-11-15 22:13:59 +01:00
parent 38cc53890a
commit da59622dce
66 changed files with 5042 additions and 3860 deletions
--- a/packages/db/src/services/clients.service.ts
+++ b/packages/db/src/services/clients.service.ts
@@ -1,4 +1,4 @@
-import { cacheable } from '@openpanel/redis';
+import { cacheable, cacheableLru } from '@openpanel/redis';
 import type { Client, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';

@@ -34,4 +34,7 @@ export async function getClientById(
  });
 }

-export const getClientByIdCached = cacheable(getClientById, 60 * 60 * 24);
+export const getClientByIdCached = cacheableLru(getClientById, {
+  maxSize: 1000,
+  ttl: 60 * 5,
+});
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -19,12 +19,9 @@ import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 import { type SqlBuilderObject, createSqlBuilder } from '../sql-builder';
 import { getEventFiltersWhereClause } from './chart.service';
-import { getOrganizationByProjectIdCached } from './organization.service';
 import type { IServiceProfile, IServiceUpsertProfile } from './profile.service';
 import {
  getProfileById,
-  getProfileByIdCached,
-  getProfiles,
  getProfilesCached,
  upsertProfile,
 } from './profile.service';
@@ -156,8 +153,6 @@ export interface IServiceEvent {
  properties: Record<string, unknown> & {
    hash?: string;
    query?: Record<string, unknown>;
-    __reqId?: string;
-    __user_agent?: string;
  };
  createdAt: Date;
  country?: string | undefined;
@@ -343,7 +338,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
    sdk_version: payload.sdkVersion ?? '',
  };

-  await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);
+  const promises = [sessionBuffer.add(event), eventBuffer.add(event)];

  if (payload.profileId) {
    const profile: IServiceUpsertProfile = {
@@ -374,10 +369,12 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
      profile.isExternal ||
      (profile.isExternal === false && payload.name === 'session_start')
    ) {
-      await upsertProfile(profile, true);
+      promises.push(upsertProfile(profile, true));
    }
  }

+  await Promise.all(promises);
+
  return {
    document: event,
  };
@@ -395,6 +392,7 @@ export interface GetEventListOptions {
  endDate?: Date;
  select?: SelectHelper<IServiceEvent>;
  custom?: (sb: SqlBuilderObject) => void;
+  dateIntervalInDays?: number;
 }

 export async function getEventList(options: GetEventListOptions) {
@@ -408,28 +406,28 @@ export async function getEventList(options: GetEventListOptions) {
    filters,
    startDate,
    endDate,
-    select: incomingSelect,
    custom,
+    select: incomingSelect,
+    dateIntervalInDays = 0.5,
  } = options;
  const { sb, getSql, join } = createSqlBuilder();

-  const organization = await getOrganizationByProjectIdCached(projectId);
-  // This will speed up the query quite a lot for big organizations
-  const dateIntervalInDays =
-    organization?.subscriptionPeriodEventsLimit &&
-    organization?.subscriptionPeriodEventsLimit > 1_000_000
-      ? 1
-      : 7;
+  const MAX_DATE_INTERVAL_IN_DAYS = 365;
+  // Cap the date interval to prevent infinity
+  const safeDateIntervalInDays = Math.min(
+    dateIntervalInDays,
+    MAX_DATE_INTERVAL_IN_DAYS,
+  );

  if (typeof cursor === 'number') {
    sb.offset = Math.max(0, (cursor ?? 0) * take);
  } else if (cursor instanceof Date) {
-    sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(cursor))}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
+    sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(cursor))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;
    sb.where.cursor = `created_at <= ${sqlstring.escape(formatClickhouseDate(cursor))}`;
  }

  if (!cursor) {
-    sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
+    sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;
  }

  sb.limit = take;
@@ -453,6 +451,9 @@ export async function getEventList(options: GetEventListOptions) {
    incomingSelect ?? {},
  );

+  sb.select.createdAt = 'created_at';
+  sb.select.projectId = 'project_id';
+
  if (select.id) {
    sb.select.id = 'id';
  }
@@ -474,9 +475,6 @@ export async function getEventList(options: GetEventListOptions) {
  if (select.properties) {
    sb.select.properties = 'properties';
  }
-  if (select.createdAt) {
-    sb.select.createdAt = 'created_at';
-  }
  if (select.country) {
    sb.select.country = 'country';
  }
@@ -583,21 +581,20 @@ export async function getEventList(options: GetEventListOptions) {
    custom(sb);
  }

-  console.log('getSql()', getSql());
-
  const data = await getEvents(getSql(), {
    profile: select.profile ?? true,
    meta: select.meta ?? true,
  });

  // If we dont get any events, try without the cursor window
-  if (data.length === 0 && sb.where.cursorWindow) {
+  if (
+    data.length === 0 &&
+    sb.where.cursorWindow &&
+    safeDateIntervalInDays < MAX_DATE_INTERVAL_IN_DAYS
+  ) {
    return getEventList({
      ...options,
-      custom(sb) {
-        options.custom?.(sb);
-        delete sb.where.cursorWindow;
-      },
+      dateIntervalInDays: dateIntervalInDays * 2,
    });
  }

@@ -945,7 +942,7 @@ class EventService {
    ]);

    if (event?.profileId) {
-      const profile = await getProfileByIdCached(event?.profileId, projectId);
+      const profile = await getProfileById(event?.profileId, projectId);
      if (profile) {
        event.profile = profile;
      }
--- a/packages/db/src/services/notification.service.ts
+++ b/packages/db/src/services/notification.service.ts
@@ -13,7 +13,7 @@ import type {
  IServiceCreateEventPayload,
  IServiceEvent,
 } from './event.service';
-import { getProfileById, getProfileByIdCached } from './profile.service';
+import { getProfileById } from './profile.service';
 import { getProjectByIdCached } from './project.service';

 type ICreateNotification = Pick<
@@ -264,10 +264,7 @@ export async function checkNotificationRulesForEvent(
    payload.profileId &&
    rules.some((rule) => rule.template?.match(/{{profile\.[^}]*}}/))
  ) {
-    const profile = await getProfileByIdCached(
-      payload.profileId,
-      payload.projectId,
-    );
+    const profile = await getProfileById(payload.profileId, payload.projectId);
    if (profile) {
      (payload as any).profile = profile;
    }
--- a/packages/db/src/services/profile.service.ts
+++ b/packages/db/src/services/profile.service.ts
@@ -106,6 +106,11 @@ export async function getProfileById(id: string, projectId: string) {
    return null;
  }

+  const cachedProfile = await profileBuffer.fetchFromCache(id, projectId);
+  if (cachedProfile) {
+    return transformProfile(cachedProfile);
+  }
+
  const [profile] = await chQuery<IClickhouseProfile>(
    `SELECT 
      id, 
@@ -127,8 +132,6 @@ export async function getProfileById(id: string, projectId: string) {
  return transformProfile(profile);
 }

-export const getProfileByIdCached = cacheable(getProfileById, 60 * 30);
-
 interface GetProfileListOptions {
  projectId: string;
  take: number;
@@ -306,10 +309,5 @@ export async function upsertProfile(
    is_external: isExternal,
  };

-  if (!isFromEvent) {
-    // Save to cache directly since the profile might be used before its saved in clickhouse
-    getProfileByIdCached.set(id, projectId)(transformProfile(profile));
-  }
-
  return profileBuffer.add(profile, isFromEvent);
 }
--- a/packages/db/src/services/salt.service.ts
+++ b/packages/db/src/services/salt.service.ts
@@ -1,6 +1,6 @@
 import { generateSalt } from '@openpanel/common/server';

-import { getRedisCache } from '@openpanel/redis';
+import { cacheableLru } from '@openpanel/redis';
 import { db } from '../prisma-client';

 export async function getCurrentSalt() {
@@ -17,36 +17,36 @@ export async function getCurrentSalt() {
  return salt.salt;
 }

-export async function getSalts() {
-  const cache = await getRedisCache().get('op:salt');
-  if (cache) {
-    return JSON.parse(cache);
-  }
+export const getSalts = cacheableLru(
+  'op:salt',
+  async () => {
+    const [curr, prev] = await db.salt.findMany({
+      orderBy: {
+        createdAt: 'desc',
+      },
+      take: 2,
+    });

-  const [curr, prev] = await db.salt.findMany({
-    orderBy: {
-      createdAt: 'desc',
-    },
-    take: 2,
-  });
+    if (!curr) {
+      throw new Error('No salt found');
+    }

-  if (!curr) {
-    throw new Error('No salt found');
-  }
+    if (!prev) {
+      throw new Error('No salt found');
+    }

-  if (!prev) {
-    throw new Error('No salt found');
-  }
+    const salts = {
+      current: curr.salt,
+      previous: prev.salt,
+    };

-  const salts = {
-    current: curr.salt,
-    previous: prev.salt,
-  };
-
-  await getRedisCache().set('op:salt', JSON.stringify(salts), 'EX', 60 * 10);
-
-  return salts;
-}
+    return salts;
+  },
+  {
+    maxSize: 2,
+    ttl: 60 * 5,
+  },
+);

 export async function createInitialSalts() {
  const MAX_RETRIES = 5;