fix: overall perf improvements

* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert

packages/trpc/src/routers/chart.ts

@@ -212,7 +212,6 @@ export const chartRouter = createTRPCRouter({
         'origin',
         'referrer',
         'referrer_name',
-        'duration',
         'created_at',
         'country',
         'city',

packages/trpc/src/routers/event.ts

@@ -127,23 +127,20 @@ export const eventRouter = createTRPCRouter({
         startDate: z.date().optional(),
         endDate: z.date().optional(),
         events: z.array(z.string()).optional(),
+        columnVisibility: z.record(z.string(), z.boolean()).optional(),
       }),
     )
-    .query(async ({ input }) => {
+    .query(async ({ input: { columnVisibility, ...input } }) => {
       const items = await getEventList({
         ...input,
         take: 50,
         cursor: input.cursor ? new Date(input.cursor) : undefined,
         select: {
-          profile: true,
-          properties: true,
-          sessionId: true,
-          deviceId: true,
-          profileId: true,
-          referrerName: true,
-          referrerType: true,
-          referrer: true,
-          origin: true,
+          ...columnVisibility,
+          city: columnVisibility?.country ?? true,
+          path: columnVisibility?.name ?? true,
+          duration: columnVisibility?.name ?? true,
+          projectId: false,
         },
       });
 
@@ -191,9 +188,10 @@ export const eventRouter = createTRPCRouter({
         startDate: z.date().optional(),
         endDate: z.date().optional(),
         events: z.array(z.string()).optional(),
+        columnVisibility: z.record(z.string(), z.boolean()).optional(),
       }),
     )
-    .query(async ({ input }) => {
+    .query(async ({ input: { columnVisibility, ...input } }) => {
       const conversions = await getConversionEventNames(input.projectId);
       const filteredConversions = conversions.filter((event) => {
         if (input.events && input.events.length > 0) {
@@ -216,15 +214,11 @@ export const eventRouter = createTRPCRouter({
         take: 50,
         cursor: input.cursor ? new Date(input.cursor) : undefined,
         select: {
-          profile: true,
-          properties: true,
-          sessionId: true,
-          deviceId: true,
-          profileId: true,
-          referrerName: true,
-          referrerType: true,
-          referrer: true,
-          origin: true,
+          ...columnVisibility,
+          city: columnVisibility?.country ?? true,
+          path: columnVisibility?.name ?? true,
+          duration: columnVisibility?.name ?? true,
+          projectId: false,
         },
         custom: (sb) => {
           sb.where.name = `name IN (${filteredConversions.map((event) => sqlstring.escape(event.name)).join(',')})`;

packages/trpc/src/routers/profile.ts

@@ -6,7 +6,7 @@ import {
   TABLE_NAMES,
   chQuery,
   createSqlBuilder,
-  getProfileByIdCached,
+  getProfileById,
   getProfileList,
   getProfileListCount,
   getProfileMetrics,
@@ -19,7 +19,7 @@ export const profileRouter = createTRPCRouter({
   byId: protectedProcedure
     .input(z.object({ profileId: z.string(), projectId: z.string() }))
     .query(async ({ input: { profileId, projectId } }) => {
-      return getProfileByIdCached(profileId, projectId);
+      return getProfileById(profileId, projectId);
     }),
 
   metrics: protectedProcedure

packages/trpc/src/routers/realtime.ts

@@ -62,10 +62,12 @@ export const realtimeRouter = createTRPCRouter({
           path: string;
           count: number;
           avg_duration: number;
+          unique_sessions: number;
         }>([
           'origin',
           'path',
           'COUNT(*) as count',
+          'COUNT(DISTINCT session_id) as unique_sessions',
           'round(avg(duration)/1000, 2) as avg_duration',
         ])
         .from(TABLE_NAMES.events)
@@ -91,9 +93,11 @@ export const realtimeRouter = createTRPCRouter({
           referrer_name: string;
           count: number;
           avg_duration: number;
+          unique_sessions: number;
         }>([
           'referrer_name',
           'COUNT(*) as count',
+          'COUNT(DISTINCT session_id) as unique_sessions',
           'round(avg(duration)/1000, 2) as avg_duration',
         ])
         .from(TABLE_NAMES.events)
@@ -120,10 +124,12 @@ export const realtimeRouter = createTRPCRouter({
           city: string;
           count: number;
           avg_duration: number;
+          unique_sessions: number;
         }>([
           'country',
           'city',
           'COUNT(*) as count',
+          'COUNT(DISTINCT session_id) as unique_sessions',
           'round(avg(duration)/1000, 2) as avg_duration',
         ])
         .from(TABLE_NAMES.events)