feat: dashboard v2, esm, upgrades (#211)

* esm

* wip

* wip

* wip

* wip

* wip

* wip

* subscription notice

* wip

* wip

* wip

* fix envs

* fix: update docker build

* fix

* esm/types

* delete dashboard :D

* add patches to dockerfiles

* update packages + catalogs + ts

* wip

* remove native libs

* ts

* improvements

* fix redirects and fetching session

* try fix favicon

* fixes

* fix

* order and resize reportds within a dashboard

* improvements

* wip

* added userjot to dashboard

* fix

* add op

* wip

* different cache key

* improve date picker

* fix table

* event details loading

* redo onboarding completely

* fix login

* fix

* fix

* extend session, billing and improve bars

* fix

* reduce price on 10M

apps/worker/Dockerfile

@@ -1,4 +1,4 @@
-ARG NODE_VERSION=20.15.1
+ARG NODE_VERSION=22.20.0
 
 FROM node:${NODE_VERSION}-slim AS base
 
@@ -38,6 +38,7 @@ COPY packages/common/package.json ./packages/common/
 COPY packages/constants/package.json ./packages/constants/
 COPY packages/validation/package.json ./packages/validation/
 COPY packages/integrations/package.json packages/integrations/
+COPY patches ./patches
 
 # BUILD
 FROM base AS build
@@ -84,6 +85,7 @@ COPY --from=build /app/packages/queue ./packages/queue
 COPY --from=build /app/packages/common ./packages/common
 COPY --from=build /app/packages/validation ./packages/validation
 COPY --from=build /app/packages/integrations ./packages/integrations
+COPY --from=build /app/tooling/typescript ./tooling/typescript
 RUN pnpm db:codegen
 
 WORKDIR /app/apps/worker

apps/worker/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@openpanel/worker",
   "version": "0.0.3",
+  "type": "module",
   "scripts": {
     "test": "vitest",
-    "dev": "dotenv -e ../../.env -c -v WATCH=1 tsup",
+    "dev": "dotenv -e ../../.env -c -v WATCH=1 tsdown",
     "testing": "WORKER_PORT=9999 pnpm dev",
     "start": "node dist/index.js",
-    "build": "rm -rf dist && tsup",
+    "build": "rm -rf dist && tsdown",
     "typecheck": "tsc --noEmit",
     "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./src/referrers/index.ts"
   },
@@ -37,7 +38,7 @@
     "@types/source-map-support": "^0.5.10",
     "@types/sqlstring": "^2.3.2",
     "@types/uuid": "^9.0.8",
-    "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "tsdown": "^0.14.2",
+    "typescript": "catalog:"
   }
 }

apps/worker/scripts/get-referrers.ts

@@ -1,5 +1,10 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 
 // extras
 const extraReferrers = {

apps/worker/src/boot-cron.ts

@@ -37,8 +37,7 @@ export async function bootCron() {
   ];
 
   if (
-    (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true' ||
-      process.env.SELF_HOSTED) &&
+    (process.env.VITE_SELF_HOSTED === 'true' || process.env.SELF_HOSTED) &&
     process.env.NODE_ENV === 'production'
   ) {
     jobs.push({

apps/worker/src/index.ts

@@ -1,5 +1,5 @@
 import { createBullBoard } from '@bull-board/api';
-import { BullMQAdapter } from '@bull-board/api/bullMQAdapter';
+import { BullMQAdapter } from '@bull-board/api/dist/src/queueAdapters/bullMQ.js';
 import { ExpressAdapter } from '@bull-board/express';
 import express from 'express';
 
@@ -13,6 +13,8 @@ import {
 } from '@openpanel/queue';
 import client from 'prom-client';
 
+import { getRedisQueue } from '@openpanel/redis';
+import { Queue, Worker } from 'bullmq';
 import { BullBoardGroupMQAdapter } from 'groupmq';
 import sourceMapSupport from 'source-map-support';
 import { bootCron } from './boot-cron';
@@ -22,6 +24,28 @@ import { logger } from './utils/logger';
 
 sourceMapSupport.install();
 
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+const test = new Worker(
+  'duplicateEvents',
+  async (job) => {
+    await sleep(5000);
+
+    return 'done';
+  },
+  {
+    connection: getRedisQueue(),
+  },
+);
+
+const testQueue = new Queue('duplicateEvents', {
+  connection: getRedisQueue(),
+  defaultJobOptions: {
+    removeOnComplete: {
+      age: 10,
+    },
+  },
+});
+
 async function start() {
   const collectDefaultMetrics = client.collectDefaultMetrics;
   collectDefaultMetrics({ register });
@@ -46,6 +70,33 @@ async function start() {
     app.use('/', serverAdapter.getRouter());
   }
 
+  const hashPayload = (payload: Record<string, unknown>) => {
+    return 'event-hash-here...';
+  };
+
+  app.get('/test', async (req, res) => {
+    try {
+      const job = await testQueue.add(
+        'test',
+        {
+          message: 'Test job!',
+        },
+        {
+          jobId: hashPayload(req.body),
+        },
+      );
+      res.json({ jobId: job.id, opts: job.opts, status: await job.getState() });
+    } catch (error) {
+      console.log('error', error);
+
+      if (error instanceof Error) {
+        res.status(500).json({ error: error.message });
+      } else {
+        res.status(500).json({ error: 'Unknown error' });
+      }
+    }
+  });
+
   app.get('/metrics', (req, res) => {
     res.set('Content-Type', register.contentType);
     register

apps/worker/src/jobs/cron.delete-projects.ts

@@ -2,7 +2,7 @@ import { logger } from '@/utils/logger';
 import { TABLE_NAMES, ch, db } from '@openpanel/db';
 import type { CronQueuePayload } from '@openpanel/queue';
 import type { Job } from 'bullmq';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 
 export async function deleteProjects(job: Job<CronQueuePayload>) {
   const projects = await db.project.findMany({
@@ -33,7 +33,7 @@ export async function deleteProjects(job: Job<CronQueuePayload>) {
     job.log(`Delete project: "${project.id}"`);
   });
 
-  const where = `project_id IN (${projects.map((project) => escape(project.id)).join(',')})`;
+  const where = `project_id IN (${projects.map((project) => sqlstring.escape(project.id)).join(',')})`;
   const tables = [
     TABLE_NAMES.events,
     TABLE_NAMES.profiles,
@@ -47,7 +47,7 @@ export async function deleteProjects(job: Job<CronQueuePayload>) {
 
   for (const table of tables) {
     const query =
-      process.env.NEXT_PUBLIC_SELF_HOSTED === 'true'
+      process.env.VITE_SELF_HOSTED === 'true'
         ? `ALTER TABLE ${table} DELETE WHERE ${where};`
         : `ALTER TABLE ${table}_replicated ON CLUSTER '{cluster}' DELETE WHERE ${where};`;
 

apps/worker/src/jobs/cron.ping.ts

@@ -12,7 +12,8 @@ export async function ping() {
         'Content-Type': 'application/json',
       },
       body: JSON.stringify({
-        domain: process.env.NEXT_PUBLIC_DASHBOARD_URL,
+        domain:
+          process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL,
         count: res?.count,
       }),
     });

apps/worker/tsdown.config.ts

@@ -0,0 +1,23 @@
+import { defineConfig } from 'tsdown';
+import type { Options } from 'tsdown';
+
+const options: Options = {
+  clean: true,
+  entry: ['src/index.ts'],
+  noExternal: [/^@openpanel\/.*$/u, /^@\/.*$/u],
+  external: ['@hyperdx/node-opentelemetry', 'winston', '@node-rs/argon2'],
+  sourcemap: true,
+  platform: 'node',
+  shims: true,
+  inputOptions: {
+    jsx: 'react',
+  },
+};
+
+if (process.env.WATCH) {
+  options.watch = ['src', '../../packages'];
+  options.onSuccess = 'node --enable-source-maps dist/index.js';
+  options.minify = false;
+}
+
+export default defineConfig(options);

apps/worker/tsup.config.ts

@@ -1,19 +0,0 @@
-import { defineConfig } from 'tsup';
-import type { Options } from 'tsup';
-
-const options: Options = {
-  clean: true,
-  entry: ['src/index.ts'],
-  noExternal: [/^@openpanel\/.*$/u, /^@\/.*$/u],
-  external: ['@hyperdx/node-opentelemetry', 'winston'],
-  ignoreWatch: ['../../**/{.git,node_modules,dist}/**'],
-  sourcemap: true,
-  splitting: false,
-};
-
-if (process.env.WATCH) {
-  options.watch = ['src/**/*', '../../packages/**/*'];
-  options.onSuccess = 'node dist/index.js';
-}
-
-export default defineConfig(options);