feature(dashboard): refactor overview

fix(lint)

apps/worker/src/boot-cron.ts

@@ -29,6 +29,11 @@ export async function bootCron() {
       type: 'flushProfiles',
       pattern: 1000 * 60,
     },
+    {
+      name: 'flush',
+      type: 'flushSessions',
+      pattern: 1000 * 10,
+    },
   ];
 
   if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {

apps/worker/src/jobs/cron.delete-projects.ts

@@ -26,6 +26,9 @@ export async function deleteProjects() {
 
   await ch.command({
     query: `DELETE FROM ${TABLE_NAMES.events} WHERE project_id IN (${projects.map((project) => escape(project.id)).join(',')});`,
+    clickhouse_settings: {
+      lightweight_deletes_sync: 0,
+    },
   });
 
   logger.info(`Deleted ${projects.length} projects`, {

apps/worker/src/jobs/cron.ts

@@ -1,6 +1,6 @@
 import type { Job } from 'bullmq';
 
-import { eventBuffer, profileBuffer } from '@openpanel/db';
+import { eventBuffer, profileBuffer, sessionBuffer } from '@openpanel/db';
 import type { CronQueuePayload } from '@openpanel/queue';
 
 import { deleteProjects } from './cron.delete-projects';
@@ -18,6 +18,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
     case 'flushProfiles': {
       return await profileBuffer.tryFlush();
     }
+    case 'flushSessions': {
+      return await sessionBuffer.tryFlush();
+    }
     case 'ping': {
       return await ping();
     }

apps/worker/src/metrics.ts

@@ -1,6 +1,12 @@
 import client from 'prom-client';
 
-import { botBuffer, db, eventBuffer, profileBuffer } from '@openpanel/db';
+import {
+  botBuffer,
+  db,
+  eventBuffer,
+  profileBuffer,
+  sessionBuffer,
+} from '@openpanel/db';
 import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';
 
 const Registry = client.Registry;
@@ -98,3 +104,14 @@ register.registerMetric(
     },
   }),
 );
+
+register.registerMetric(
+  new client.Gauge({
+    name: `buffer_${sessionBuffer.name}_count`,
+    help: 'Number of unprocessed sessions',
+    async collect() {
+      const metric = await sessionBuffer.getBufferSize();
+      this.set(metric);
+    },
+  }),
+);

apps/worker/src/utils/session-handler.ts

@@ -77,6 +77,21 @@ export async function getSessionEnd({
       throw new Error('Invalid session end job');
     }
 
+    // If the profile_id is set and it's different from the device_id, we need to update the profile_id
+    if (
+      sessionEnd.job.data.payload.profileId !== profileId &&
+      sessionEnd.job.data.payload.profileId ===
+        sessionEnd.job.data.payload.deviceId
+    ) {
+      await sessionEnd.job.updateData({
+        ...sessionEnd.job.data,
+        payload: {
+          ...sessionEnd.job.data.payload,
+          profileId,
+        },
+      });
+    }
+
     await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
   }