feat: insights

* fix: migration for newly created self-hosting instances

* fix: build script

* wip

* wip

* wip

* fix: tailwind css

apps/worker/src/jobs/cron.ts

@@ -6,6 +6,7 @@ import type { CronQueuePayload } from '@openpanel/queue';
 import { jobdeleteProjects } from './cron.delete-projects';
 import { ping } from './cron.ping';
 import { salt } from './cron.salt';
+import { insightsDailyJob } from './insights';
 
 export async function cronJob(job: Job<CronQueuePayload>) {
   switch (job.data.type) {
@@ -27,5 +28,8 @@ export async function cronJob(job: Job<CronQueuePayload>) {
     case 'deleteProjects': {
       return await jobdeleteProjects(job);
     }
+    case 'insightsDaily': {
+      return await insightsDailyJob(job);
+    }
   }
 }

apps/worker/src/jobs/insights.ts

@@ -0,0 +1,72 @@
+import { ch } from '@openpanel/db/src/clickhouse/client';
+import {
+  createEngine,
+  devicesModule,
+  entryPagesModule,
+  geoModule,
+  insightStore,
+  pageTrendsModule,
+  referrersModule,
+} from '@openpanel/db/src/services/insights';
+import type {
+  CronQueuePayload,
+  InsightsQueuePayloadProject,
+} from '@openpanel/queue';
+import { insightsQueue } from '@openpanel/queue';
+import type { Job } from 'bullmq';
+
+const defaultEngineConfig = {
+  keepTopNPerModuleWindow: 20,
+  closeStaleAfterDays: 7,
+  dimensionBatchSize: 50,
+  globalThresholds: {
+    minTotal: 200,
+    minAbsDelta: 80,
+    minPct: 0.15,
+  },
+};
+
+export async function insightsDailyJob(job: Job<CronQueuePayload>) {
+  const projectIds = await insightStore.listProjectIdsForCadence('daily');
+  const date = new Date().toISOString().slice(0, 10);
+
+  for (const projectId of projectIds) {
+    await insightsQueue.add(
+      'insightsProject',
+      {
+        type: 'insightsProject',
+        payload: { projectId, date },
+      },
+      {
+        jobId: `daily:${date}:${projectId}`, // idempotent
+      },
+    );
+  }
+}
+
+export async function insightsProjectJob(
+  job: Job<InsightsQueuePayloadProject>,
+) {
+  const { projectId, date } = job.data.payload;
+  const engine = createEngine({
+    store: insightStore,
+    modules: [
+      referrersModule,
+      entryPagesModule,
+      pageTrendsModule,
+      geoModule,
+      devicesModule,
+    ],
+    db: ch,
+    config: defaultEngineConfig,
+  });
+
+  const projectCreatedAt = await insightStore.getProjectCreatedAt(projectId);
+
+  await engine.runProject({
+    projectId,
+    cadence: 'daily',
+    now: new Date(date),
+    projectCreatedAt,
+  });
+}