feat: dashboard v2, esm, upgrades (#211)

* esm

* wip

* wip

* wip

* wip

* wip

* wip

* subscription notice

* wip

* wip

* wip

* fix envs

* fix: update docker build

* fix

* esm/types

* delete dashboard :D

* add patches to dockerfiles

* update packages + catalogs + ts

* wip

* remove native libs

* ts

* improvements

* fix redirects and fetching session

* try fix favicon

* fixes

* fix

* order and resize reportds within a dashboard

* improvements

* wip

* added userjot to dashboard

* fix

* add op

* wip

* different cache key

* improve date picker

* fix table

* event details loading

* redo onboarding completely

* fix login

* fix

* fix

* extend session, billing and improve bars

* fix

* reduce price on 10M

packages/auth/constants.ts

@@ -1,6 +1,8 @@
 import { parseCookieDomain } from './parse-cookie-domain';
 
-const parsed = parseCookieDomain(process.env.NEXT_PUBLIC_DASHBOARD_URL ?? '');
+const parsed = parseCookieDomain(
+  (process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL) ?? '',
+);
 
 export const COOKIE_MAX_AGE = 60 * 60 * 24 * 30;
 export const COOKIE_OPTIONS = {

packages/auth/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/auth",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -15,13 +16,13 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
-    "@types/react": "^18.2.0",
+    "@types/node": "catalog:",
+    "@types/react": "catalog:",
     "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   },
   "peerDependencies": {
     "next": "14.2.1",
-    "react": "18.2.0"
+    "react": "catalog:"
   }
 }

packages/auth/parse-cookie-domain.ts

@@ -111,14 +111,3 @@ export const parseCookieDomain = (url: string) => {
     secure: domain.protocol === 'https:',
   };
 };
-
-const parsed = parseCookieDomain(process.env.NEXT_PUBLIC_DASHBOARD_URL ?? '');
-
-export const COOKIE_MAX_AGE = 60 * 60 * 24 * 30;
-export const COOKIE_OPTIONS = {
-  domain: parsed.domain,
-  secure: parsed.secure,
-  sameSite: 'lax',
-  httpOnly: true,
-  path: '/',
-} as const;

packages/cli/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/cli",
   "version": "0.0.1-beta",
+  "type": "module",
   "module": "index.ts",
   "bin": {
     "openpanel": "dist/bin/cli.js"
@@ -23,10 +24,10 @@
     "@openpanel/db": "workspace:^",
     "@openpanel/sdk": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "@types/progress": "^2.0.7",
     "@types/ramda": "^0.30.1",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/cli/src/importer/importer.ts

@@ -336,7 +336,7 @@ async function sendBatchToAPI(
         'openpanel-client-id': clientId,
         'openpanel-client-secret': clientSecret,
       },
-      body: zlib.gzipSync(JSON.stringify(batch)),
+      body: Buffer.from(zlib.gzipSync(JSON.stringify(batch))),
     });
     if (!res.ok) {
       throw new Error(`Failed to send batch: ${await res.text()}`);

packages/common/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/common",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "test": "vitest",
@@ -22,10 +23,10 @@
     "@openpanel/tsconfig": "workspace:*",
     "@openpanel/validation": "workspace:*",
     "@types/luxon": "^3.6.2",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "@types/ramda": "^0.29.6",
     "@types/ua-parser-js": "^0.7.39",
     "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/constants/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/constants",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -8,7 +9,6 @@
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
     "date-fns": "^3.3.1",
-    "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/db/code-migrations/2-accounts.ts

@@ -1,5 +1,10 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import { db } from '../index';
 import { printBoxMessage } from './helpers';
 

packages/db/code-migrations/3-init-ch.ts

@@ -1,5 +1,10 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import { formatClickhouseDate } from '../src/clickhouse/client';
 import {
   createDatabase,

packages/db/code-migrations/helpers.ts

@@ -24,9 +24,7 @@ export function getIsCluster() {
 }
 
 export function getIsSelfHosting() {
-  return (
-    process.env.NEXT_PUBLIC_SELF_HOSTED === 'true' || !!process.env.SELF_HOSTED
-  );
+  return process.env.VITE_SELF_HOSTED === 'true' || !!process.env.SELF_HOSTED;
 }
 
 export function getIsDry() {

packages/db/code-migrations/migrate.ts

@@ -1,5 +1,10 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import { db } from '../index';
 import { getIsDry, getIsSelfHosting, printBoxMessage } from './helpers';
 

packages/db/package.json

@@ -1,9 +1,10 @@
 {
   "name": "@openpanel/db",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
-    "codegen": "pnpm with-env prisma generate",
+    "codegen": "pnpm with-env prisma generate && jiti prisma/prisma-json-types.ts",
     "migrate": "pnpm with-env prisma migrate dev",
     "migrate:deploy:code": "pnpm with-env jiti ./code-migrations/migrate.ts",
     "migrate:deploy:db": "pnpm with-env prisma migrate deploy",
@@ -20,8 +21,8 @@
     "@openpanel/queue": "workspace:^",
     "@openpanel/redis": "workspace:*",
     "@openpanel/validation": "workspace:*",
-    "@prisma/client": "^5.1.1",
-    "@prisma/extension-read-replicas": "^0.4.0",
+    "@prisma/client": "^6.14.0",
+    "@prisma/extension-read-replicas": "^0.4.1",
     "fast-deep-equal": "^3.1.3",
     "jiti": "^2.4.1",
     "prisma-json-types-generator": "^3.1.1",
@@ -33,11 +34,11 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "@types/ramda": "^0.29.6",
     "@types/sqlstring": "^2.3.2",
     "@types/uuid": "^9.0.8",
-    "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "prisma": "^6.14.0",
+    "typescript": "catalog:"
   }
 }

packages/db/prisma/migrations/20251013121758_report_layout/migration.sql

@@ -0,0 +1,29 @@
+-- AlterTable
+ALTER TABLE "public"."_IntegrationToNotificationRule" ADD CONSTRAINT "_IntegrationToNotificationRule_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_IntegrationToNotificationRule_AB_unique";
+
+-- CreateTable
+CREATE TABLE "public"."report_layouts" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "reportId" UUID NOT NULL,
+    "x" INTEGER NOT NULL DEFAULT 0,
+    "y" INTEGER NOT NULL DEFAULT 0,
+    "w" INTEGER NOT NULL DEFAULT 4,
+    "h" INTEGER NOT NULL DEFAULT 3,
+    "minW" INTEGER DEFAULT 2,
+    "minH" INTEGER DEFAULT 2,
+    "maxW" INTEGER,
+    "maxH" INTEGER,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "report_layouts_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "report_layouts_reportId_key" ON "public"."report_layouts"("reportId");
+
+-- AddForeignKey
+ALTER TABLE "public"."report_layouts" ADD CONSTRAINT "report_layouts_reportId_fkey" FOREIGN KEY ("reportId") REFERENCES "public"."reports"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/db/prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
-provider = "postgresql"
+# It should be added in your version-control system (e.g., Git)
+provider = "postgresql"

packages/db/prisma/prisma-json-types.ts

@@ -0,0 +1,210 @@
+import { readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+interface JsonFieldMapping {
+  model: string;
+  field: string;
+  type: string;
+}
+
+function parseSchemaForJsonTypes(schemaPath: string): JsonFieldMapping[] {
+  const schemaContent = readFileSync(schemaPath, 'utf-8');
+  const lines = schemaContent.split('\n');
+  const mappings: JsonFieldMapping[] = [];
+
+  let currentModel = '';
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]?.trim() || '';
+
+    // Track current model
+    if (line.startsWith('model ')) {
+      const parts = line.split(' ');
+      currentModel = parts[1] || '';
+      continue;
+    }
+
+    // Look for Json fields with type comments
+    if (line.includes('Json') && i > 0) {
+      const prevLine = lines[i - 1]?.trim() || '';
+      const typeMatch = prevLine.match(/\/\/\/ \[([^\]]+)\]/);
+
+      if (typeMatch) {
+        const fieldMatch = line.match(/(\w+)\s+Json/);
+        if (fieldMatch?.[1] && typeMatch[1]) {
+          mappings.push({
+            model: currentModel,
+            field: fieldMatch[1],
+            type: typeMatch[1],
+          });
+        }
+      }
+    }
+  }
+
+  return mappings;
+}
+
+function processGeneratedFiles(
+  generatedDir: string,
+  mappings: JsonFieldMapping[],
+): void {
+  // Process the main files in the generated directory
+  const mainFiles = [
+    'client.ts',
+    'commonInputTypes.ts',
+    'enums.ts',
+    'models.ts',
+  ];
+
+  for (const fileName of mainFiles) {
+    const filePath = join(generatedDir, fileName);
+    try {
+      replaceJsonValueInFileForModel(filePath, mappings);
+    } catch (error) {
+      console.log(`Skipping ${filePath}: ${error}`);
+    }
+  }
+
+  // Process files in the models subdirectory - each file corresponds to one model
+  const modelsDir = join(generatedDir, 'models');
+  try {
+    const modelFiles = readdirSync(modelsDir);
+    for (const fileName of modelFiles) {
+      if (fileName.endsWith('.ts')) {
+        const filePath = join(modelsDir, fileName);
+        try {
+          // Extract model name from filename (e.g., "Notification.ts" -> "Notification")
+          const modelName = fileName.replace('.ts', '');
+
+          // Only process mappings for this specific model
+          const modelMappings = mappings.filter((m) => m.model === modelName);
+
+          if (modelMappings.length > 0) {
+            replaceJsonValueInFileForModel(filePath, modelMappings);
+          }
+        } catch (error) {
+          console.log(`Skipping ${filePath}: ${error}`);
+        }
+      }
+    }
+  } catch (error) {
+    console.log(`Could not read models directory: ${error}`);
+  }
+}
+
+function replaceJsonValueInFileForModel(
+  filePath: string,
+  mappings: JsonFieldMapping[],
+): void {
+  let content = readFileSync(filePath, 'utf-8');
+  let modified = false;
+
+  for (const mapping of mappings) {
+    // Pattern 1: Simple runtime.JsonValue replacement (for select/return types)
+    const simpleJsonValueRegex = new RegExp(
+      `\\b${mapping.field}:\\s*runtime\\.JsonValue\\b`,
+      'g',
+    );
+    if (simpleJsonValueRegex.test(content)) {
+      content = content.replace(
+        simpleJsonValueRegex,
+        `${mapping.field}: PrismaJson.${mapping.type}`,
+      );
+      modified = true;
+    }
+
+    // Pattern 2: runtime.InputJsonValue with optional JsonNullValueInput (for create/update inputs)
+    const inputJsonValueRegex = new RegExp(
+      `\\b${mapping.field}:\\s*(?:Prisma\\.JsonNullValueInput\\s*\\|\\s*)?runtime\\.InputJsonValue\\b`,
+      'g',
+    );
+    if (inputJsonValueRegex.test(content)) {
+      content = content.replace(
+        inputJsonValueRegex,
+        `${mapping.field}: PrismaJson.${mapping.type}`,
+      );
+      modified = true;
+    }
+
+    // Pattern 3: Optional runtime.InputJsonValue with optional JsonNullValueInput
+    const optionalInputJsonValueRegex = new RegExp(
+      `\\b${mapping.field}\\?:\\s*(?:Prisma\\.JsonNullValueInput\\s*\\|\\s*)?runtime\\.InputJsonValue\\b`,
+      'g',
+    );
+    if (optionalInputJsonValueRegex.test(content)) {
+      content = content.replace(
+        optionalInputJsonValueRegex,
+        `${mapping.field}?: PrismaJson.${mapping.type}`,
+      );
+      modified = true;
+    }
+
+    // Pattern 4: Union types with JsonNullValueInput | runtime.InputJsonValue
+    const unionJsonValueRegex =
+      /(Prisma\.JsonNullValueInput\s*\|\s*)runtime\.InputJsonValue/g;
+    if (unionJsonValueRegex.test(content)) {
+      content = content.replace(
+        unionJsonValueRegex,
+        `$1PrismaJson.${mapping.type}`,
+      );
+      modified = true;
+    }
+
+    // Pattern 5: Just runtime.InputJsonValue in unions
+    const simpleInputJsonValueRegex = /\|\s*runtime\.InputJsonValue/g;
+    if (simpleInputJsonValueRegex.test(content)) {
+      content = content.replace(
+        simpleInputJsonValueRegex,
+        `| PrismaJson.${mapping.type}`,
+      );
+      modified = true;
+    }
+
+    // Pattern 6: Optional union types with JsonNullValueInput | runtime.InputJsonValue
+    const optionalUnionJsonValueRegex = new RegExp(
+      `\\b${mapping.field}\\?:\\s*(?:Prisma\\.JsonNullValueInput\\s*\\|\\s*)?runtime\\.InputJsonValue\\b`,
+      'g',
+    );
+    if (optionalUnionJsonValueRegex.test(content)) {
+      content = content.replace(
+        optionalUnionJsonValueRegex,
+        `${mapping.field}?: PrismaJson.${mapping.type}`,
+      );
+      modified = true;
+    }
+  }
+
+  if (modified) {
+    writeFileSync(filePath, content, 'utf-8');
+    console.log(`Updated ${filePath}`);
+  }
+}
+
+function main() {
+  const schemaPath = join(__dirname, '../prisma/schema.prisma');
+  const generatedDir = join(__dirname, '../src/generated/prisma');
+
+  console.log('Parsing schema for Json type mappings...');
+  const mappings = parseSchemaForJsonTypes(schemaPath);
+
+  console.log('Found Json type mappings:');
+  mappings.forEach((m) => console.log(`  ${m.model}.${m.field} -> ${m.type}`));
+
+  if (mappings.length === 0) {
+    console.log('No mappings found!');
+    return;
+  }
+
+  console.log('Processing generated files...');
+  processGeneratedFiles(generatedDir, mappings);
+
+  console.log('Post-codegen script completed!');
+}
+
+main();

packages/db/prisma/schema.prisma

@@ -2,12 +2,16 @@
 // learn more about it in the docs: https://pris.ly/d/prisma-schema
 
 generator client {
-  provider = "prisma-client-js"
+  provider               = "prisma-client"
+  output                 = "../src/generated/prisma"
+  moduleFormat           = "esm"
+  generatedFileExtension = "ts"
+  importFileExtension    = "ts"
 }
 
-generator json {
-  provider = "prisma-json-types-generator"
-}
+// generator json {
+//   provider = "prisma-json-types-generator"
+// }
 
 datasource db {
   provider  = "postgresql"
@@ -315,7 +319,8 @@ model Report {
   funnelWindow Float?
 
   dashboardId String
-  dashboard   Dashboard @relation(fields: [dashboardId], references: [id], onDelete: Cascade)
+  dashboard   Dashboard     @relation(fields: [dashboardId], references: [id], onDelete: Cascade)
+  layout      ReportLayout?
 
   createdAt DateTime @default(now())
   updatedAt DateTime @default(now()) @updatedAt
@@ -323,6 +328,29 @@ model Report {
   @@map("reports")
 }
 
+model ReportLayout {
+  id       String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+  reportId String @unique @db.Uuid
+  report   Report @relation(fields: [reportId], references: [id], onDelete: Cascade)
+
+  // Grid position and size
+  x Int @default(0)
+  y Int @default(0)
+  w Int @default(4)
+  h Int @default(3)
+
+  // Optional: store additional layout preferences
+  minW Int? @default(2)
+  minH Int? @default(2)
+  maxW Int?
+  maxH Int?
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @default(now()) @updatedAt
+
+  @@map("report_layouts")
+}
+
 model ShareOverview {
   id             String       @unique
   projectId      String       @unique

packages/db/src/clickhouse/client.ts

@@ -1,6 +1,6 @@
 import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client';
 import { ClickHouseLogLevel, createClient } from '@clickhouse/client';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 
 import type { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config';
 import { createLogger } from '@openpanel/logger';
@@ -201,14 +201,14 @@ export function toDate(str: string, interval?: IInterval) {
   // If it does not match the regex it's a column name eg 'created_at'
   if (!interval || interval === 'minute' || interval === 'hour') {
     if (str.match(/\d{4}-\d{2}-\d{2}/)) {
-      return escape(str);
+      return sqlstring.escape(str);
     }
 
     return str;
   }
 
   if (str.match(/\d{4}-\d{2}-\d{2}/)) {
-    return `toDate(${escape(str.split(' ')[0])})`;
+    return `toDate(${sqlstring.escape(str.split(' ')[0])})`;
   }
 
   return `toDate(${str})`;

packages/db/src/clickhouse/query-builder.ts

@@ -1,6 +1,6 @@
 import type { ClickHouseClient, ResponseJSON } from '@clickhouse/client';
 import type { IInterval } from '@openpanel/validation';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 
 type SqlValue = string | number | boolean | Date | null | Expression;
 type SqlParam = SqlValue | SqlValue[];
@@ -133,7 +133,7 @@ export class Query<T = any> {
       return this.escapeDate(value);
     }
 
-    return escape(value);
+    return sqlstring.escape(value);
   }
 
   where(column: string, operator: Operator, value?: SqlParam): this {
@@ -258,11 +258,11 @@ export class Query<T = any> {
 
   private escapeDate(value: string | Date): string {
     if (value instanceof Date) {
-      return escape(clix.datetime(value));
+      return sqlstring.escape(clix.datetime(value));
     }
 
     return value.replaceAll(this._dateRegex, (match) => {
-      return escape(match);
+      return sqlstring.escape(match);
     });
   }
 

packages/db/src/prisma-client.ts

@@ -1,8 +1,8 @@
 import { createLogger } from '@openpanel/logger';
-import { type Organization, PrismaClient } from '@prisma/client';
 import { readReplicas } from '@prisma/extension-read-replicas';
+import { type Organization, PrismaClient } from './generated/prisma/client';
 
-export * from '@prisma/client';
+export * from './generated/prisma/client';
 
 const logger = createLogger({ name: 'db' });
 
@@ -59,7 +59,7 @@ const getPrismaClient = () => {
           subscriptionStatus: {
             needs: { subscriptionStatus: true, subscriptionCanceledAt: true },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return 'active';
               }
 
@@ -69,7 +69,7 @@ const getPrismaClient = () => {
           hasSubscription: {
             needs: { subscriptionStatus: true, subscriptionEndsAt: true },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return false;
               }
 
@@ -94,7 +94,7 @@ const getPrismaClient = () => {
               subscriptionPeriodEventsCountExceededAt: true,
             },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return null;
               }
 
@@ -131,7 +131,7 @@ const getPrismaClient = () => {
           isCanceled: {
             needs: { subscriptionStatus: true, subscriptionCanceledAt: true },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return false;
               }
 
@@ -145,7 +145,7 @@ const getPrismaClient = () => {
               subscriptionEndsAt: true,
             },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return false;
               }
 
@@ -159,7 +159,7 @@ const getPrismaClient = () => {
               subscriptionCanceledAt: true,
             },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return false;
               }
 
@@ -182,7 +182,7 @@ const getPrismaClient = () => {
               subscriptionPeriodEventsLimit: true,
             },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return false;
               }
 
@@ -195,7 +195,7 @@ const getPrismaClient = () => {
           subscriptionCurrentPeriodStart: {
             needs: { subscriptionStartsAt: true, subscriptionInterval: true },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return null;
               }
 
@@ -229,7 +229,7 @@ const getPrismaClient = () => {
               subscriptionInterval: true,
             },
             compute(org) {
-              if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
+              if (process.env.VITE_SELF_HOSTED === 'true') {
                 return null;
               }
 

packages/db/src/services/chart.service.ts

@@ -1,4 +1,4 @@
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 
 import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common';
 import type {
@@ -45,7 +45,7 @@ export function getSelectPropertyKey(property: string) {
   if (!match) return property;
 
   if (property.includes('*')) {
-    return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${escape(
+    return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(${match}, ${sqlstring.escape(
       transformPropertyKey(property),
     )})))`;
   }
@@ -76,11 +76,11 @@ export function getChartSql({
   } = createSqlBuilder();
 
   sb.where = getEventFiltersWhereClause(event.filters);
-  sb.where.projectId = `project_id = ${escape(projectId)}`;
+  sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
 
   if (event.name !== '*') {
-    sb.select.label_0 = `${escape(event.name)} as label_0`;
-    sb.where.eventName = `name = ${escape(event.name)}`;
+    sb.select.label_0 = `${sqlstring.escape(event.name)} as label_0`;
+    sb.where.eventName = `name = ${sqlstring.escape(event.name)}`;
   } else {
     sb.select.label_0 = `'*' as label_0`;
   }
@@ -99,7 +99,7 @@ export function getChartSql({
       first_name as "profile.first_name",
       last_name as "profile.last_name",
       properties as "profile.properties"
-    FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${escape(projectId)}) as profile on profile.id = profile_id`;
+    FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;
   }
 
   sb.select.count = 'count(*) as count';
@@ -251,14 +251,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         case 'is': {
           if (isWildcard) {
             where[id] = `arrayExists(x -> ${value
-              .map((val) => `x = ${escape(String(val).trim())}`)
+              .map((val) => `x = ${sqlstring.escape(String(val).trim())}`)
               .join(' OR ')}, ${whereFrom})`;
           } else {
             if (value.length === 1) {
-              where[id] = `${whereFrom} = ${escape(String(value[0]).trim())}`;
+              where[id] =
+                `${whereFrom} = ${sqlstring.escape(String(value[0]).trim())}`;
             } else {
               where[id] = `${whereFrom} IN (${value
-                .map((val) => escape(String(val).trim()))
+                .map((val) => sqlstring.escape(String(val).trim()))
                 .join(', ')})`;
             }
           }
@@ -267,14 +268,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         case 'isNot': {
           if (isWildcard) {
             where[id] = `arrayExists(x -> ${value
-              .map((val) => `x != ${escape(String(val).trim())}`)
+              .map((val) => `x != ${sqlstring.escape(String(val).trim())}`)
               .join(' OR ')}, ${whereFrom})`;
           } else {
             if (value.length === 1) {
-              where[id] = `${whereFrom} != ${escape(String(value[0]).trim())}`;
+              where[id] =
+                `${whereFrom} != ${sqlstring.escape(String(value[0]).trim())}`;
             } else {
               where[id] = `${whereFrom} NOT IN (${value
-                .map((val) => escape(String(val).trim()))
+                .map((val) => sqlstring.escape(String(val).trim()))
                 .join(', ')})`;
             }
           }
@@ -283,13 +285,16 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         case 'contains': {
           if (isWildcard) {
             where[id] = `arrayExists(x -> ${value
-              .map((val) => `x LIKE ${escape(`%${String(val).trim()}%`)}`)
+              .map(
+                (val) =>
+                  `x LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
+              )
               .join(' OR ')}, ${whereFrom})`;
           } else {
             where[id] = `(${value
               .map(
                 (val) =>
-                  `${whereFrom} LIKE ${escape(`%${String(val).trim()}%`)}`,
+                  `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
               )
               .join(' OR ')})`;
           }
@@ -298,13 +303,16 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         case 'doesNotContain': {
           if (isWildcard) {
             where[id] = `arrayExists(x -> ${value
-              .map((val) => `x NOT LIKE ${escape(`%${String(val).trim()}%`)}`)
+              .map(
+                (val) =>
+                  `x NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
+              )
               .join(' OR ')}, ${whereFrom})`;
           } else {
             where[id] = `(${value
               .map(
                 (val) =>
-                  `${whereFrom} NOT LIKE ${escape(`%${String(val).trim()}%`)}`,
+                  `${whereFrom} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
               )
               .join(' OR ')})`;
           }
@@ -313,13 +321,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         case 'startsWith': {
           if (isWildcard) {
             where[id] = `arrayExists(x -> ${value
-              .map((val) => `x LIKE ${escape(`${String(val).trim()}%`)}`)
+              .map(
+                (val) => `x LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
+              )
               .join(' OR ')}, ${whereFrom})`;
           } else {
             where[id] = `(${value
               .map(
                 (val) =>
-                  `${whereFrom} LIKE ${escape(`${String(val).trim()}%`)}`,
+                  `${whereFrom} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
               )
               .join(' OR ')})`;
           }
@@ -328,13 +338,15 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         case 'endsWith': {
           if (isWildcard) {
             where[id] = `arrayExists(x -> ${value
-              .map((val) => `x LIKE ${escape(`%${String(val).trim()}`)}`)
+              .map(
+                (val) => `x LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
+              )
               .join(' OR ')}, ${whereFrom})`;
           } else {
             where[id] = `(${value
               .map(
                 (val) =>
-                  `${whereFrom} LIKE ${escape(`%${String(val).trim()}`)}`,
+                  `${whereFrom} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
               )
               .join(' OR ')})`;
           }
@@ -343,12 +355,13 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         case 'regex': {
           if (isWildcard) {
             where[id] = `arrayExists(x -> ${value
-              .map((val) => `match(x, ${escape(String(val).trim())})`)
+              .map((val) => `match(x, ${sqlstring.escape(String(val).trim())})`)
               .join(' OR ')}, ${whereFrom})`;
           } else {
             where[id] = `(${value
               .map(
-                (val) => `match(${whereFrom}, ${escape(String(val).trim())})`,
+                (val) =>
+                  `match(${whereFrom}, ${sqlstring.escape(String(val).trim())})`,
               )
               .join(' OR ')})`;
           }
@@ -376,10 +389,11 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
       switch (operator) {
         case 'is': {
           if (value.length === 1) {
-            where[id] = `${name} = ${escape(String(value[0]).trim())}`;
+            where[id] =
+              `${name} = ${sqlstring.escape(String(value[0]).trim())}`;
           } else {
             where[id] = `${name} IN (${value
-              .map((val) => escape(String(val).trim()))
+              .map((val) => sqlstring.escape(String(val).trim()))
               .join(', ')})`;
           }
           break;
@@ -394,37 +408,48 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
         }
         case 'isNot': {
           if (value.length === 1) {
-            where[id] = `${name} != ${escape(String(value[0]).trim())}`;
+            where[id] =
+              `${name} != ${sqlstring.escape(String(value[0]).trim())}`;
           } else {
             where[id] = `${name} NOT IN (${value
-              .map((val) => escape(String(val).trim()))
+              .map((val) => sqlstring.escape(String(val).trim()))
               .join(', ')})`;
           }
           break;
         }
         case 'contains': {
           where[id] = `(${value
-            .map((val) => `${name} LIKE ${escape(`%${String(val).trim()}%`)}`)
+            .map(
+              (val) =>
+                `${name} LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
+            )
             .join(' OR ')})`;
           break;
         }
         case 'doesNotContain': {
           where[id] = `(${value
             .map(
-              (val) => `${name} NOT LIKE ${escape(`%${String(val).trim()}%`)}`,
+              (val) =>
+                `${name} NOT LIKE ${sqlstring.escape(`%${String(val).trim()}%`)}`,
             )
             .join(' OR ')})`;
           break;
         }
         case 'startsWith': {
           where[id] = `(${value
-            .map((val) => `${name} LIKE ${escape(`${String(val).trim()}%`)}`)
+            .map(
+              (val) =>
+                `${name} LIKE ${sqlstring.escape(`${String(val).trim()}%`)}`,
+            )
             .join(' OR ')})`;
           break;
         }
         case 'endsWith': {
           where[id] = `(${value
-            .map((val) => `${name} LIKE ${escape(`%${String(val).trim()}`)}`)
+            .map(
+              (val) =>
+                `${name} LIKE ${sqlstring.escape(`%${String(val).trim()}`)}`,
+            )
             .join(' OR ')})`;
           break;
         }
@@ -432,7 +457,7 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
           where[id] = `(${value
             .map(
               (val) =>
-                `match(${name}, ${escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
+                `match(${name}, ${sqlstring.escape(stripLeadingAndTrailingSlashes(String(val)).trim())})`,
             )
             .join(' OR ')})`;
           break;

packages/db/src/services/event.service.ts

@@ -1,5 +1,5 @@
 import { path, assocPath, last, mergeDeepRight } from 'ramda';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 import { v4 as uuid } from 'uuid';
 
 import { DateTime, toDots } from '@openpanel/common';
@@ -17,10 +17,17 @@ import {
 import { type Query, clix } from '../clickhouse/query-builder';
 import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
-import { createSqlBuilder } from '../sql-builder';
+import { type SqlBuilderObject, createSqlBuilder } from '../sql-builder';
 import { getEventFiltersWhereClause } from './chart.service';
+import { getOrganizationByProjectIdCached } from './organization.service';
 import type { IServiceProfile, IServiceUpsertProfile } from './profile.service';
-import { getProfileById, getProfiles, upsertProfile } from './profile.service';
+import {
+  getProfileById,
+  getProfileByIdCached,
+  getProfiles,
+  getProfilesCached,
+  upsertProfile,
+} from './profile.service';
 
 export type IImportedEvent = Omit<
   IClickhouseEvent,
@@ -258,7 +265,7 @@ export async function getEvents(
     const ids = events
       .filter((e) => e.device_id !== e.profile_id)
       .map((e) => e.profile_id);
-    const profiles = await getProfiles(ids, projectId);
+    const profiles = await getProfilesCached(ids, projectId);
 
     const map = new Map<string, IServiceProfile>();
     for (const profile of profiles) {
@@ -266,7 +273,17 @@ export async function getEvents(
     }
 
     for (const event of events) {
-      event.profile = map.get(event.profile_id);
+      event.profile = map.get(event.profile_id) ?? {
+        id: event.profile_id,
+        email: '',
+        avatar: '',
+        firstName: '',
+        lastName: '',
+        createdAt: new Date(),
+        projectId,
+        isExternal: false,
+        properties: {},
+      };
     }
   }
 
@@ -365,6 +382,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
 export interface GetEventListOptions {
   projectId: string;
   profileId?: string;
+  sessionId?: string;
   take: number;
   cursor?: number | Date;
   events?: string[] | null;
@@ -372,29 +390,46 @@ export interface GetEventListOptions {
   startDate?: Date;
   endDate?: Date;
   select?: SelectHelper<IServiceEvent>;
+  custom?: (sb: SqlBuilderObject) => void;
 }
 
-export async function getEventList({
-  cursor,
-  take,
-  projectId,
-  profileId,
-  events,
-  filters,
-  startDate,
-  endDate,
-  select: incomingSelect,
-}: GetEventListOptions) {
+export async function getEventList(options: GetEventListOptions) {
+  const {
+    cursor,
+    take,
+    projectId,
+    profileId,
+    sessionId,
+    events,
+    filters,
+    startDate,
+    endDate,
+    select: incomingSelect,
+    custom,
+  } = options;
   const { sb, getSql, join } = createSqlBuilder();
 
+  const organization = await getOrganizationByProjectIdCached(projectId);
+  // This will speed up the query quite a lot for big organizations
+  const dateIntervalInDays =
+    organization?.subscriptionPeriodEventsLimit &&
+    organization?.subscriptionPeriodEventsLimit > 1_000_000
+      ? 1
+      : 7;
+
   if (typeof cursor === 'number') {
     sb.offset = Math.max(0, (cursor ?? 0) * take);
   } else if (cursor instanceof Date) {
-    sb.where.cursor = `created_at <= '${formatClickhouseDate(cursor)}'`;
+    sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(cursor))}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
+    sb.where.cursor = `created_at <= ${sqlstring.escape(formatClickhouseDate(cursor))}`;
+  }
+
+  if (!cursor) {
+    sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
   }
 
   sb.limit = take;
-  sb.where.projectId = `project_id = ${escape(projectId)}`;
+  sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
   const select = mergeDeepRight(
     {
       id: true,
@@ -503,7 +538,11 @@ export async function getEventList({
   }
 
   if (profileId) {
-    sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND device_id != '' AND profile_id = ${escape(profileId)} group by did) OR profile_id = ${escape(profileId)})`;
+    sb.where.deviceId = `(device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND device_id != '' AND profile_id = ${sqlstring.escape(profileId)} group by did) OR profile_id = ${sqlstring.escape(profileId)})`;
+  }
+
+  if (sessionId) {
+    sb.where.sessionId = `session_id = ${sqlstring.escape(sessionId)}`;
   }
 
   if (startDate && endDate) {
@@ -527,10 +566,29 @@ export async function getEventList({
   sb.orderBy.created_at =
     'toDate(created_at) DESC, created_at DESC, profile_id DESC, name DESC';
 
-  return getEvents(getSql(), {
+  if (custom) {
+    custom(sb);
+  }
+
+  console.log('getSql()', getSql());
+
+  const data = await getEvents(getSql(), {
     profile: select.profile ?? true,
     meta: select.meta ?? true,
   });
+
+  // If we dont get any events, try without the cursor window
+  if (data.length === 0 && sb.where.cursorWindow) {
+    return getEventList({
+      ...options,
+      custom(sb) {
+        options.custom?.(sb);
+        delete sb.where.cursorWindow;
+      },
+    });
+  }
+
+  return data;
 }
 
 export const getEventsCountCached = cacheable(getEventsCount, 60 * 10);
@@ -543,9 +601,9 @@ export async function getEventsCount({
   endDate,
 }: Omit<GetEventListOptions, 'cursor' | 'take'>) {
   const { sb, getSql, join } = createSqlBuilder();
-  sb.where.projectId = `project_id = ${escape(projectId)}`;
+  sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
   if (profileId) {
-    sb.where.profileId = `profile_id = ${escape(profileId)}`;
+    sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
   }
 
   if (startDate && endDate) {
@@ -614,7 +672,7 @@ export async function getTopPages({
     SELECT path, count(*) as count, project_id, first_value(created_at) as first_seen, last_value(properties['__title']) as title, origin
     FROM ${TABLE_NAMES.events} 
     WHERE name = 'screen_view' 
-    AND  project_id = ${escape(projectId)} 
+    AND  project_id = ${sqlstring.escape(projectId)} 
     AND created_at > now() - INTERVAL 30 DAY 
     ${search ? `AND path ILIKE '%${search}%'` : ''}
     GROUP BY path, project_id, origin
@@ -824,36 +882,43 @@ class EventService {
     id: string;
     createdAt?: Date;
   }) {
-    const event = await clix(this.client)
-      .select<IClickhouseEvent>(['*'])
-      .from('events')
-      .where('project_id', '=', projectId)
-      .when(!!createdAt, (q) => {
-        if (createdAt) {
-          q.where('created_at', 'BETWEEN', [
-            new Date(createdAt.getTime() - 1000),
-            new Date(createdAt.getTime() + 1000),
-          ]);
-        }
-      })
-      .where('id', '=', id)
-      .limit(1)
-      .execute()
-      .then((res) => {
-        if (!res[0]) {
-          return null;
-        }
+    const [event, metas] = await Promise.all([
+      clix(this.client)
+        .select<IClickhouseEvent>(['*'])
+        .from('events')
+        .where('project_id', '=', projectId)
+        .when(!!createdAt, (q) => {
+          if (createdAt) {
+            q.where('created_at', 'BETWEEN', [
+              new Date(createdAt.getTime() - 1000),
+              new Date(createdAt.getTime() + 1000),
+            ]);
+          }
+        })
+        .where('id', '=', id)
+        .limit(1)
+        .execute()
+        .then((res) => {
+          if (!res[0]) {
+            return null;
+          }
 
-        return transformEvent(res[0]);
-      });
+          return transformEvent(res[0]);
+        }),
+      getEventMetasCached(projectId),
+    ]);
 
     if (event?.profileId) {
-      const profile = await getProfileById(event?.profileId, projectId);
+      const profile = await getProfileByIdCached(event?.profileId, projectId);
       if (profile) {
         event.profile = profile;
       }
     }
 
+    if (event) {
+      event.meta = metas.find((meta) => meta.name === event.name);
+    }
+
     return event;
   }
 

packages/db/src/services/funnel.service.ts

@@ -1,7 +1,7 @@
 import { ifNaN } from '@openpanel/common';
 import type { IChartEvent, IChartInput } from '@openpanel/validation';
 import { last, reverse } from 'ramda';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 import { ch } from '../clickhouse/client';
 import { TABLE_NAMES } from '../clickhouse/client';
 import { clix } from '../clickhouse/query-builder';
@@ -24,7 +24,7 @@ export class FunnelService {
     return events.map((event) => {
       const { sb, getWhere } = createSqlBuilder();
       sb.where = getEventFiltersWhereClause(event.filters);
-      sb.where.name = `name = ${escape(event.name)}`;
+      sb.where.name = `name = ${sqlstring.escape(event.name)}`;
       return getWhere().replace('WHERE ', '');
     });
   }

packages/db/src/services/organization.service.ts

@@ -1,6 +1,6 @@
 import { DateTime } from '@openpanel/common';
 import { cacheable } from '@openpanel/redis';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 import { chQuery, formatClickhouseDate } from '../clickhouse/client';
 import type { Invite, Prisma, ProjectAccess, User } from '../prisma-client';
 import { db } from '../prisma-client';
@@ -69,11 +69,14 @@ export async function getInvites(organizationId: string) {
     where: {
       organizationId,
     },
+    orderBy: {
+      createdAt: 'desc',
+    },
   });
 }
 
-export function getInviteById(inviteId: string) {
-  return db.invite.findUnique({
+export async function getInviteById(inviteId: string) {
+  const res = await db.invite.findUnique({
     where: {
       id: inviteId,
     },
@@ -86,6 +89,11 @@ export function getInviteById(inviteId: string) {
       },
     },
   });
+
+  return {
+    ...res,
+    isExpired: res?.expiresAt && res.expiresAt < new Date(),
+  };
 }
 
 export async function getMembers(organizationId: string) {
@@ -201,8 +209,8 @@ export async function getOrganizationBillingEventsCount(
   const { sb, getSql } = createSqlBuilder();
 
   sb.select.count = 'COUNT(*) AS count';
-  sb.where.projectIds = `project_id IN (${organization.projects.map((project) => escape(project.id)).join(',')})`;
-  sb.where.createdAt = `created_at BETWEEN ${escape(formatClickhouseDate(organization.subscriptionCurrentPeriodStart))} AND ${escape(formatClickhouseDate(organization.subscriptionCurrentPeriodEnd))}`;
+  sb.where.projectIds = `project_id IN (${organization.projects.map((project) => sqlstring.escape(project.id)).join(',')})`;
+  sb.where.createdAt = `created_at BETWEEN ${sqlstring.escape(formatClickhouseDate(organization.subscriptionCurrentPeriodStart))} AND ${sqlstring.escape(formatClickhouseDate(organization.subscriptionCurrentPeriodEnd))}`;
 
   const res = await chQuery<{ count: number }>(getSql());
   return res[0]?.count;
@@ -224,9 +232,9 @@ export async function getOrganizationBillingEventsCountSerie(
   sb.select.count = 'COUNT(*) AS count';
   sb.select.day = `toDate(toStartOf${interval.slice(0, 1).toUpperCase() + interval.slice(1)}(created_at)) AS ${interval}`;
   sb.groupBy.day = interval;
-  sb.orderBy.day = `${interval} WITH FILL FROM toDate(${escape(formatClickhouseDate(startDate, true))}) TO toDate(${escape(formatClickhouseDate(endDate, true))}) STEP INTERVAL 1 ${interval.toUpperCase()}`;
-  sb.where.projectIds = `project_id IN (${organization.projects.map((project) => escape(project.id)).join(',')})`;
-  sb.where.createdAt = `${interval} BETWEEN ${escape(formatClickhouseDate(startDate, true))} AND ${escape(formatClickhouseDate(endDate, true))}`;
+  sb.orderBy.day = `${interval} WITH FILL FROM toDate(${sqlstring.escape(formatClickhouseDate(startDate, true))}) TO toDate(${sqlstring.escape(formatClickhouseDate(endDate, true))}) STEP INTERVAL 1 ${interval.toUpperCase()}`;
+  sb.where.projectIds = `project_id IN (${organization.projects.map((project) => sqlstring.escape(project.id)).join(',')})`;
+  sb.where.createdAt = `${interval} BETWEEN ${sqlstring.escape(formatClickhouseDate(startDate, true))} AND ${sqlstring.escape(formatClickhouseDate(endDate, true))}`;
 
   const res = await chQuery<{ count: number; day: string }>(getSql());
   return res;

packages/db/src/services/profile.service.ts

@@ -1,5 +1,5 @@
 import { omit, uniq } from 'ramda';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 
 import { strip, toObject } from '@openpanel/common';
 import { cacheable } from '@openpanel/redis';
@@ -21,25 +21,69 @@ export type IProfileMetrics = {
   sessions: number;
   durationAvg: number;
   durationP90: number;
+  totalEvents: number;
+  uniqueDaysActive: number;
+  bounceRate: number;
+  avgEventsPerSession: number;
+  conversionEvents: number;
+  avgTimeBetweenSessions: number;
 };
 export function getProfileMetrics(profileId: string, projectId: string) {
   return chQuery<IProfileMetrics>(`
     WITH lastSeen AS (
-      SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
     ),
     firstSeen AS (
-      SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
     ),
     screenViews AS (
-      SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
     ),
     sessions AS (
-      SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
     ),
     duration AS (
-      SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT 
+        round(avg(duration) / 1000 / 60, 2) as durationAvg, 
+        round(quantilesExactInclusive(0.9)(duration)[1] / 1000 / 60, 2) as durationP90 
+      FROM ${TABLE_NAMES.events} 
+      WHERE name = 'session_end' AND duration != 0 AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
+    ),
+    totalEvents AS (
+      SELECT count(*) as totalEvents FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
+    ),
+    uniqueDaysActive AS (
+      SELECT count(DISTINCT toDate(created_at)) as uniqueDaysActive FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
+    ),
+    bounceRate AS (
+      SELECT round(avg(properties['__bounce'] = '1') * 100, 4) as bounceRate FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
+    ),
+    avgEventsPerSession AS (
+      SELECT round((SELECT totalEvents FROM totalEvents) / nullIf((SELECT sessions FROM sessions), 0), 2) as avgEventsPerSession
+    ),
+    conversionEvents AS (
+      SELECT count(*) as conversionEvents FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
+    ),
+    avgTimeBetweenSessions AS (
+      SELECT 
+        CASE 
+          WHEN (SELECT sessions FROM sessions) <= 1 THEN 0
+          ELSE round(dateDiff('second', (SELECT firstSeen FROM firstSeen), (SELECT lastSeen FROM lastSeen)) / nullIf((SELECT sessions FROM sessions) - 1, 0), 1)
+        END as avgTimeBetweenSessions
     )
-    SELECT lastSeen, firstSeen, screenViews, sessions, durationAvg, durationP90 FROM lastSeen, firstSeen, screenViews,sessions, duration
+    SELECT 
+      (SELECT lastSeen FROM lastSeen) as lastSeen, 
+      (SELECT firstSeen FROM firstSeen) as firstSeen, 
+      (SELECT screenViews FROM screenViews) as screenViews, 
+      (SELECT sessions FROM sessions) as sessions, 
+      (SELECT durationAvg FROM duration) as durationAvg, 
+      (SELECT durationP90 FROM duration) as durationP90,
+      (SELECT totalEvents FROM totalEvents) as totalEvents,
+      (SELECT uniqueDaysActive FROM uniqueDaysActive) as uniqueDaysActive,
+      (SELECT bounceRate FROM bounceRate) as bounceRate,
+      (SELECT avgEventsPerSession FROM avgEventsPerSession) as avgEventsPerSession,
+      (SELECT conversionEvents FROM conversionEvents) as conversionEvents,
+      (SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions
   `).then((data) => data[0]!);
 }
 
@@ -59,7 +103,7 @@ export async function getProfileById(id: string, projectId: string) {
       last_value(is_external) as is_external, 
       last_value(properties) as properties, 
       last_value(created_at) as created_at
-    FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
+    FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,
   );
 
   if (!profile) {
@@ -77,6 +121,7 @@ interface GetProfileListOptions {
   cursor?: number;
   filters?: IChartEventFilter[];
   search?: string;
+  isExternal?: boolean;
 }
 
 export async function getProfiles(ids: string[], projectId: string) {
@@ -99,8 +144,8 @@ export async function getProfiles(ids: string[], projectId: string) {
       any(created_at) as created_at
     FROM ${TABLE_NAMES.profiles}
     WHERE 
-      project_id = ${escape(projectId)} AND
-      id IN (${filteredIds.map((id) => escape(id)).join(',')})
+      project_id = ${sqlstring.escape(projectId)} AND
+      id IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')})
     GROUP BY id, project_id
     `,
   );
@@ -108,36 +153,48 @@ export async function getProfiles(ids: string[], projectId: string) {
   return data.map(transformProfile);
 }
 
+export const getProfilesCached = cacheable(getProfiles, 60 * 5);
+
 export async function getProfileList({
   take,
   cursor,
   projectId,
-  filters,
   search,
+  isExternal,
 }: GetProfileListOptions) {
   const { sb, getSql } = createSqlBuilder();
   sb.from = `${TABLE_NAMES.profiles} FINAL`;
   sb.select.all = '*';
-  sb.where.project_id = `project_id = ${escape(projectId)}`;
+  sb.where.project_id = `project_id = ${sqlstring.escape(projectId)}`;
   sb.limit = take;
   sb.offset = Math.max(0, (cursor ?? 0) * take);
   sb.orderBy.created_at = 'created_at DESC';
   if (search) {
     sb.where.search = `(email ILIKE '%${search}%' OR first_name ILIKE '%${search}%' OR last_name ILIKE '%${search}%')`;
   }
+  if (isExternal !== undefined) {
+    sb.where.external = `is_external = ${isExternal ? 'true' : 'false'}`;
+  }
   const data = await chQuery<IClickhouseProfile>(getSql());
   return data.map(transformProfile);
 }
 
 export async function getProfileListCount({
   projectId,
-  filters,
+  isExternal,
+  search,
 }: Omit<GetProfileListOptions, 'cursor' | 'take'>) {
   const { sb, getSql } = createSqlBuilder();
   sb.from = 'profiles';
   sb.select.count = 'count(id) as count';
-  sb.where.project_id = `project_id = ${escape(projectId)}`;
+  sb.where.project_id = `project_id = ${sqlstring.escape(projectId)}`;
   sb.groupBy.project_id = 'project_id';
+  if (search) {
+    sb.where.search = `(email ILIKE '%${search}%' OR first_name ILIKE '%${search}%' OR last_name ILIKE '%${search}%')`;
+  }
+  if (isExternal !== undefined) {
+    sb.where.external = `is_external = ${isExternal ? 'true' : 'false'}`;
+  }
   const data = await chQuery<{ count: number }>(getSql());
   return data[0]?.count ?? 0;
 }

packages/db/src/services/project.service.ts

@@ -1,5 +1,5 @@
 import { cacheable } from '@openpanel/redis';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 import { TABLE_NAMES, chQuery } from '../clickhouse/client';
 import type { Prisma, Project } from '../prisma-client';
 import { db } from '../prisma-client';
@@ -50,7 +50,7 @@ export async function getProjectsByOrganizationId(organizationId: string) {
       organizationId,
     },
     orderBy: {
-      createdAt: 'desc',
+      eventsCount: 'desc',
     },
   });
 }
@@ -104,7 +104,7 @@ export async function getProjects({
 
 export const getProjectEventsCount = async (projectId: string) => {
   const res = await chQuery<{ count: number }>(
-    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`,
+    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)}`,
   );
   return res[0]?.count;
 };

packages/db/src/services/reference.service.ts

@@ -16,19 +16,3 @@ export async function getReferenceById(id: string) {
 
   return reference;
 }
-
-export async function getReferences({
-  where,
-  take,
-  skip,
-}: {
-  where: Prisma.ReferenceWhereInput;
-  take?: number;
-  skip?: number;
-}) {
-  return db.reference.findMany({
-    where,
-    take: take ?? 50,
-    skip,
-  });
-}

packages/db/src/services/reports.service.ts

@@ -14,7 +14,7 @@ import type {
 } from '@openpanel/validation';
 
 import { db } from '../prisma-client';
-import type { Report as DbReport } from '../prisma-client';
+import type { Report as DbReport, ReportLayout } from '../prisma-client';
 
 export type IServiceReport = Awaited<ReturnType<typeof getReportById>>;
 
@@ -46,8 +46,8 @@ export function transformReportEvent(
 }
 
 export function transformReport(
-  report: DbReport,
-): IChartProps & { id: string } {
+  report: DbReport & { layout?: ReportLayout | null },
+): IChartProps & { id: string; layout?: ReportLayout | null } {
   return {
     id: report.id,
     projectId: report.projectId,
@@ -68,6 +68,7 @@ export function transformReport(
     criteria: (report.criteria as ICriteria) ?? undefined,
     funnelGroup: report.funnelGroup ?? undefined,
     funnelWindow: report.funnelWindow ?? undefined,
+    layout: report.layout ?? undefined,
   };
 }
 
@@ -77,6 +78,9 @@ export function getReportsByDashboardId(dashboardId: string) {
       where: {
         dashboardId,
       },
+      include: {
+        layout: true,
+      },
     })
     .then((reports) => reports.map(transformReport));
 }
@@ -86,6 +90,9 @@ export async function getReportById(id: string) {
     where: {
       id,
     },
+    include: {
+      layout: true,
+    },
   });
 
   if (!report) {

packages/db/src/services/retention.service.ts

@@ -1,4 +1,4 @@
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 
 import { TABLE_NAMES, chQuery } from '../clickhouse/client';
 
@@ -16,7 +16,7 @@ WITH
           profile_id,
           max(toWeek(created_at)) AS last_seen
       FROM ${TABLE_NAMES.events}
-      WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)
+      WHERE (project_id = ${sqlstring.escape(projectId)}) AND (profile_id != device_id)
       GROUP BY profile_id
   ),
   n AS
@@ -25,7 +25,7 @@ WITH
           profile_id,
           min(toWeek(created_at)) AS first_seen
       FROM ${TABLE_NAMES.events}
-      WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)
+      WHERE (project_id = ${sqlstring.escape(projectId)}) AND (profile_id != device_id)
       GROUP BY profile_id
   ),
   a AS
@@ -85,7 +85,7 @@ export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {
       AND toStartOfWeek(events.created_at) = toStartOfWeek(future_events.created_at - toIntervalWeek(1))
       AND future_events.profile_id != future_events.device_id
     WHERE 
-      project_id = ${escape(projectId)} 
+      project_id = ${sqlstring.escape(projectId)} 
       AND events.profile_id != events.device_id
     GROUP BY 1
     ORDER BY date ASC`;
@@ -122,11 +122,11 @@ export function getRollingActiveUsers({
       (
           SELECT *
           FROM ${TABLE_NAMES.dau_mv}
-          WHERE project_id = ${escape(projectId)}
+          WHERE project_id = ${sqlstring.escape(projectId)}
       )
       ARRAY JOIN range(${days}) AS n
     )
-    WHERE project_id = ${escape(projectId)}
+    WHERE project_id = ${sqlstring.escape(projectId)}
     GROUP BY date`;
 
   return chQuery<IServiceRetentionRollingActiveUsers>(sql);
@@ -141,7 +141,7 @@ export function getRetentionLastSeenSeries({
             max(created_at) AS last_active,
             profile_id
         FROM ${TABLE_NAMES.events}
-        WHERE (project_id = ${escape(projectId)}) AND (device_id != profile_id)
+        WHERE (project_id = ${sqlstring.escape(projectId)}) AND (device_id != profile_id)
         GROUP BY profile_id
     )
     SELECT

packages/db/src/services/session.service.ts

@@ -1,5 +1,18 @@
-import { TABLE_NAMES, ch } from '../clickhouse/client';
+import { cacheable } from '@openpanel/redis';
+import type { IChartEventFilter } from '@openpanel/validation';
+import { uniq } from 'ramda';
+import sqlstring from 'sqlstring';
+import {
+  TABLE_NAMES,
+  ch,
+  chQuery,
+  formatClickhouseDate,
+} from '../clickhouse/client';
 import { clix } from '../clickhouse/query-builder';
+import { createSqlBuilder } from '../sql-builder';
+import { getEventFiltersWhereClause } from './chart.service';
+import { getOrganizationByProjectIdCached } from './organization.service';
+import { type IServiceProfile, getProfilesCached } from './profile.service';
 
 export type IClickhouseSession = {
   id: string;
@@ -43,17 +56,291 @@ export type IClickhouseSession = {
   properties: Record<string, string>;
 };
 
+export interface IServiceSession {
+  id: string;
+  profileId: string;
+  eventCount: number;
+  screenViewCount: number;
+  entryPath: string;
+  entryOrigin: string;
+  exitPath: string;
+  exitOrigin: string;
+  createdAt: Date;
+  endedAt: Date;
+  referrer: string;
+  referrerName: string;
+  referrerType: string;
+  os: string;
+  osVersion: string;
+  browser: string;
+  browserVersion: string;
+  device: string;
+  brand: string;
+  model: string;
+  country: string;
+  region: string;
+  city: string;
+  longitude: number | null;
+  latitude: number | null;
+  isBounce: boolean;
+  projectId: string;
+  deviceId: string;
+  duration: number;
+  utmMedium: string;
+  utmSource: string;
+  utmCampaign: string;
+  utmContent: string;
+  utmTerm: string;
+  revenue: number;
+  properties: Record<string, string>;
+  profile?: IServiceProfile;
+}
+
+export interface GetSessionListOptions {
+  projectId: string;
+  profileId?: string;
+  take: number;
+  filters?: IChartEventFilter[];
+  startDate?: Date;
+  endDate?: Date;
+  search?: string;
+  cursor?: Cursor | null;
+}
+
+export function transformSession(session: IClickhouseSession): IServiceSession {
+  return {
+    id: session.id,
+    profileId: session.profile_id,
+    eventCount: session.event_count,
+    screenViewCount: session.screen_view_count,
+    entryPath: session.entry_path,
+    entryOrigin: session.entry_origin,
+    exitPath: session.exit_path,
+    exitOrigin: session.exit_origin,
+    createdAt: new Date(session.created_at),
+    endedAt: new Date(session.ended_at),
+    referrer: session.referrer,
+    referrerName: session.referrer_name,
+    referrerType: session.referrer_type,
+    os: session.os,
+    osVersion: session.os_version,
+    browser: session.browser,
+    browserVersion: session.browser_version,
+    device: session.device,
+    brand: session.brand,
+    model: session.model,
+    country: session.country,
+    region: session.region,
+    city: session.city,
+    longitude: session.longitude,
+    latitude: session.latitude,
+    isBounce: session.is_bounce,
+    projectId: session.project_id,
+    deviceId: session.device_id,
+    duration: session.duration,
+    utmMedium: session.utm_medium,
+    utmSource: session.utm_source,
+    utmCampaign: session.utm_campaign,
+    utmContent: session.utm_content,
+    utmTerm: session.utm_term,
+    revenue: session.revenue,
+    properties: session.properties,
+    profile: undefined,
+  };
+}
+
+type Direction = 'initial' | 'next' | 'prev';
+
+type PageInfo = {
+  next?: Cursor; // use last row
+};
+
+type Cursor = {
+  createdAt: string; // ISO 8601 with ms
+  id: string;
+};
+
+export async function getSessionList({
+  cursor,
+  take,
+  projectId,
+  profileId,
+  filters,
+  startDate,
+  endDate,
+  search,
+}: GetSessionListOptions) {
+  const { sb, getSql } = createSqlBuilder();
+
+  sb.from = `${TABLE_NAMES.sessions} FINAL`;
+  sb.limit = take;
+  sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
+
+  if (startDate && endDate) {
+    sb.where.range = `created_at BETWEEN toDateTime('${formatClickhouseDate(startDate)}') AND toDateTime('${formatClickhouseDate(endDate)}')`;
+  }
+
+  if (profileId)
+    sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
+  if (search) {
+    const s = sqlstring.escape(`%${search}%`);
+    sb.where.search = `(entry_path ILIKE ${s} OR exit_path ILIKE ${s} OR referrer ILIKE ${s} OR referrer_name ILIKE ${s})`;
+  }
+  if (filters?.length) {
+    Object.assign(sb.where, getEventFiltersWhereClause(filters));
+  }
+
+  const organization = await getOrganizationByProjectIdCached(projectId);
+  // This will speed up the query quite a lot for big organizations
+  const dateIntervalInDays =
+    organization?.subscriptionPeriodEventsLimit &&
+    organization?.subscriptionPeriodEventsLimit > 1_000_000
+      ? 1
+      : 7;
+
+  if (cursor) {
+    const cAt = sqlstring.escape(cursor.createdAt);
+    const cId = sqlstring.escape(cursor.id);
+    sb.where.cursor = `(created_at < toDateTime64(${cAt}, 3) OR (created_at = toDateTime64(${cAt}, 3) AND id < ${cId}))`;
+    sb.where.cursorWindow = `created_at >= toDateTime64(${cAt}, 3) - INTERVAL ${dateIntervalInDays} DAY`;
+    sb.orderBy.created_at = 'toDate(created_at) DESC, created_at DESC, id DESC';
+  } else {
+    sb.orderBy.created_at = 'toDate(created_at) DESC, created_at DESC, id DESC';
+    sb.where.created_at = `created_at > now() - INTERVAL ${dateIntervalInDays} DAY`;
+  }
+
+  // ==== Select columns (as you had) ====
+  // sb.select.id = 'id'; sb.select.project_id = 'project_id'; ... etc.
+  const columns = [
+    'created_at',
+    'ended_at',
+    'id',
+    'profile_id',
+    'entry_path',
+    'exit_path',
+    'duration',
+    'is_bounce',
+    'referrer_name',
+    'referrer',
+    'country',
+    'city',
+    'os',
+    'browser',
+    'brand',
+    'model',
+    'device',
+    'screen_view_count',
+    'event_count',
+    'revenue',
+  ];
+
+  columns.forEach((column) => {
+    sb.select[column] = column;
+  });
+
+  const sql = getSql();
+  const data = await chQuery<
+    IClickhouseSession & {
+      latestCreatedAt: string;
+    }
+  >(sql);
+
+  // Compute cursors from page edges
+  const last = data[take - 1];
+
+  const meta: PageInfo = {
+    next: last
+      ? {
+          createdAt: last.created_at,
+          id: last.id,
+        }
+      : undefined,
+  };
+
+  // Profile hydration (unchanged)
+  const profileIds = data
+    .filter((e) => e.device_id !== e.profile_id)
+    .map((e) => e.profile_id);
+  const profiles = await getProfilesCached(profileIds, projectId);
+  const map = new Map<string, IServiceProfile>(profiles.map((p) => [p.id, p]));
+
+  const items = data.map(transformSession).map((item) => ({
+    ...item,
+    profile: map.get(item.profileId) ?? {
+      id: item.profileId,
+      email: '',
+      avatar: '',
+      firstName: '',
+      lastName: '',
+      createdAt: new Date(),
+      projectId,
+      isExternal: false,
+      properties: {},
+    },
+  }));
+
+  return { items, meta };
+}
+
+export async function getSessionsCount({
+  projectId,
+  profileId,
+  filters,
+  startDate,
+  endDate,
+  search,
+}: Omit<GetSessionListOptions, 'take' | 'cursor'>) {
+  const { sb, getSql } = createSqlBuilder();
+
+  sb.select.count = 'count(*) as count';
+  sb.where.projectId = `project_id = ${sqlstring.escape(projectId)}`;
+  sb.where.sign = 'sign = 1';
+
+  if (profileId) {
+    sb.where.profileId = `profile_id = ${sqlstring.escape(profileId)}`;
+  }
+
+  if (startDate && endDate) {
+    sb.where.created_at = `toDate(created_at) BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;
+  }
+
+  if (search) {
+    sb.where.search = `(entry_path ILIKE '%${search}%' OR exit_path ILIKE '%${search}%' OR referrer ILIKE '%${search}%' OR referrer_name ILIKE '%${search}%')`;
+  }
+
+  if (filters && filters.length > 0) {
+    const sessionFilters = getEventFiltersWhereClause(filters);
+    sb.where = {
+      ...sb.where,
+      ...sessionFilters,
+    };
+  }
+
+  sb.from = TABLE_NAMES.sessions;
+
+  const result = await chQuery<{ count: number }>(getSql());
+  return result[0]?.count ?? 0;
+}
+
+export const getSessionsCountCached = cacheable(getSessionsCount, 60 * 10);
+
 class SessionService {
   constructor(private client: typeof ch) {}
 
-  byId(sessionId: string, projectId: string) {
-    return clix(this.client)
+  async byId(sessionId: string, projectId: string) {
+    const result = await clix(this.client)
       .select<IClickhouseSession>(['*'])
       .from(TABLE_NAMES.sessions)
       .where('id', '=', sessionId)
       .where('project_id', '=', projectId)
-      .execute()
-      .then((res) => res[0]);
+      .where('sign', '=', 1)
+      .execute();
+
+    if (!result[0]) {
+      throw new Error('Session not found');
+    }
+
+    return transformSession(result[0]);
   }
 }
 

packages/email/package.json

@@ -1,24 +1,25 @@
 {
   "name": "@openpanel/email",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "dev": "email dev --dir src/emails -p 3939",
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@react-email/components": "^0.0.30",
-    "react": "18.2.0",
-    "react-dom": "18.2.0",
+    "@react-email/components": "^0.5.6",
+    "react": "catalog:",
+    "react-dom": "catalog:",
     "resend": "^4.0.1",
     "responsive-react-email": "^0.0.5",
     "zod": "catalog:"
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
-    "@types/react": "^18.2.0",
+    "@types/node": "catalog:",
+    "@types/react": "catalog:",
     "react-email": "3.0.4",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/email/src/emails/email-invite.tsx

@@ -1,5 +1,5 @@
-import React from 'react';
 import { Button, Link, Text } from '@react-email/components';
+import React from 'react';
 import { z } from 'zod';
 import { Layout } from '../components/layout';
 

packages/email/src/emails/email-reset-password.tsx

@@ -1,5 +1,5 @@
-import React from 'react';
 import { Button, Link, Text } from '@react-email/components';
+import React from 'react';
 import { z } from 'zod';
 import { Layout } from '../components/layout';
 

packages/geo/package.json

@@ -11,10 +11,10 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "fast-extract": "^1.4.3",
     "tar": "^7.4.3",
-    "typescript": "^5.2.2",
+    "typescript": "catalog:",
     "jiti": "^2.4.1"
   }
 }

packages/geo/scripts/download.ts

@@ -1,6 +1,11 @@
 import fs from 'node:fs';
 import https from 'node:https';
 import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import zlib from 'node:zlib';
 import * as tar from 'tar';
 import type { Parser } from 'tar';

packages/geo/src/geo.ts

@@ -1,5 +1,10 @@
 import { readFile } from 'node:fs/promises';
 import path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 import type { ReaderModel } from '@maxmind/geoip2-node';
 import { Reader } from '@maxmind/geoip2-node';
 

packages/integrations/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/integrations",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -12,7 +13,7 @@
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
     "@openpanel/validation": "workspace:*",
-    "@types/node": "20.14.8",
-    "typescript": "^5.2.2"
+    "@types/node": "catalog:",
+    "typescript": "catalog:"
   }
 }

packages/integrations/src/slack.ts

@@ -1,6 +1,7 @@
 // Cred to (@c_alares) https://github.com/christianalares/seventy-seven/blob/main/packages/integrations/src/slack/index.ts
 
-import { LogLevel, App as SlackApp } from '@slack/bolt';
+import * as Slack from '@slack/bolt';
+const { LogLevel, App: SlackApp } = Slack;
 import { InstallProvider } from '@slack/oauth';
 
 const SLACK_CLIENT_ID = process.env.SLACK_CLIENT_ID;

packages/json/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/json",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -11,7 +12,7 @@
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
     "@openpanel/validation": "workspace:*",
-    "@types/node": "20.14.8",
-    "typescript": "^5.2.2"
+    "@types/node": "catalog:",
+    "typescript": "catalog:"
   }
 }

packages/logger/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/logger",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -13,6 +14,6 @@
     "@openpanel/tsconfig": "workspace:*",
     "date-fns": "^3.3.1",
     "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/payments/package.json

@@ -1,22 +1,23 @@
 {
   "name": "@openpanel/payments",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@polar-sh/sdk": "^0.26.1"
+    "@polar-sh/sdk": "^0.35.4"
   },
   "devDependencies": {
     "@openpanel/db": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",
     "@types/inquirer": "^9.0.7",
     "@types/inquirer-autocomplete-prompt": "^3.0.3",
-    "@types/node": "20.14.8",
-    "@types/react": "^18.2.0",
+    "@types/node": "catalog:",
+    "@types/react": "catalog:",
     "inquirer": "^9.3.5",
     "inquirer-autocomplete-prompt": "^3.0.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/payments/scripts/create-custom-pricing.ts

@@ -1,10 +1,8 @@
 import { db } from '@openpanel/db';
 import { Polar } from '@polar-sh/sdk';
-import type { ProductCreate } from '@polar-sh/sdk/models/components/productcreate';
 import inquirer from 'inquirer';
 import inquirerAutocomplete from 'inquirer-autocomplete-prompt';
-import { PRICING, getProducts, getSuccessUrl, polar } from '..';
-import { formatEventsCount } from './create-products';
+import { getSuccessUrl } from '..';
 
 // Register the autocomplete prompt
 inquirer.registerPrompt('autocomplete', inquirerAutocomplete);
@@ -222,6 +220,7 @@ async function main() {
   });
 
   const checkoutLink = await polar.checkoutLinks.create({
+    paymentProcessor: 'stripe',
     productId: product.id,
     allowDiscountCodes: false,
     metadata: {

packages/payments/scripts/create-products.ts

@@ -1,5 +1,6 @@
 import { Polar } from '@polar-sh/sdk';
-import type { ProductCreate } from '@polar-sh/sdk/models/components/productcreate';
+import type { Product } from '@polar-sh/sdk/dist/esm/models/components/product';
+import type { ProductCreate } from '@polar-sh/sdk/dist/esm/models/components/productcreate';
 import inquirer from 'inquirer';
 import { PRICING } from '../';
 
@@ -69,7 +70,7 @@ async function main() {
 
   const isDry = process.argv.includes('--dry');
   const products = await getProducts();
-  const createProducts = [];
+  const createProducts: Product[] = [];
   for (const price of PRICING) {
     if (price.price === 0) {
       const exists = products.find(

packages/payments/src/polar.ts

@@ -50,13 +50,13 @@ export async function createPortal({
 }
 
 export async function createCheckout({
-  priceId,
+  productId,
   organizationId,
   projectId,
   user,
   ipAddress,
 }: {
-  priceId: string;
+  productId: string;
   organizationId: string;
   projectId?: string;
   user: {
@@ -68,9 +68,10 @@ export async function createCheckout({
   ipAddress: string;
 }) {
   return polar.checkouts.create({
-    productPriceId: priceId,
+    // productPriceId: priceId,
+    products: [productId],
     successUrl: getSuccessUrl(
-      process.env.NEXT_PUBLIC_DASHBOARD_URL!,
+      process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!,
       organizationId,
       projectId,
     ),
@@ -90,7 +91,6 @@ export async function cancelSubscription(subscriptionId: string) {
       id: subscriptionId,
       subscriptionUpdate: {
         cancelAtPeriodEnd: true,
-        revoke: null,
       },
     });
   } catch (error) {
@@ -110,7 +110,6 @@ export function reactivateSubscription(subscriptionId: string) {
     id: subscriptionId,
     subscriptionUpdate: {
       cancelAtPeriodEnd: false,
-      revoke: null,
     },
   });
 }

packages/payments/src/prices.ts

@@ -26,7 +26,7 @@ export const PRICING: IPrice[] = [
   },
   { price: 180, events: 2_500_000 },
   { price: 250, events: 5_000_000 },
-  { price: 400, events: 10_000_000 },
+  { price: 300, events: 10_000_000 },
 ];
 
 export const FREE_PRODUCT_IDS = [

packages/payments/tsconfig.json

@@ -1,12 +1,16 @@
 {
-  "extends": "@openpanel/tsconfig/base.json",
   "compilerOptions": {
-    "baseUrl": ".",
-    "paths": {
-      "@/*": ["./src/*"]
-    },
-    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "resolveJsonModule": true,
+    "allowJs": true
   },
-  "include": ["."],
-  "exclude": ["node_modules"]
+  "include": ["src/**/*"]
 }

packages/queue/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/queue",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -15,7 +16,7 @@
   "devDependencies": {
     "@openpanel/sdk": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
-    "typescript": "^5.2.2"
+    "@types/node": "catalog:",
+    "typescript": "catalog:"
   }
 }

packages/redis/package.json

@@ -1,19 +1,20 @@
 {
   "name": "@openpanel/redis",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
     "@openpanel/json": "workspace:*",
-    "ioredis": "^5.4.1"
+    "ioredis": "^5.7.0"
   },
   "devDependencies": {
     "@openpanel/db": "workspace:*",
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/redis/redis.ts

@@ -1,6 +1,6 @@
 import { getSuperJson, setSuperJson } from '@openpanel/json';
 import type { RedisOptions } from 'ioredis';
-import Redis from 'ioredis';
+import { Redis } from 'ioredis';
 
 const options: RedisOptions = {
   connectTimeout: 10000,

packages/sdks/_info/package.json

@@ -6,12 +6,12 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "react": "18.2.0"
+    "react": "catalog:"
   },
   "devDependencies": {
-    "@types/react": "^18.2.20",
+    "@types/react": "catalog:",
     "@openpanel/tsconfig": "workspace:*",
     "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/sdks/astro/src/OpenPanelComponent.astro

@@ -1,8 +1,5 @@
 ---
-import type {
-OpenPanelMethodNames,
-OpenPanelOptions
-} from '@openpanel/web';
+import type { OpenPanelMethodNames, OpenPanelOptions } from '@openpanel/web';
 
 type Props = Omit<OpenPanelOptions, 'filter'> & {
   profileId?: string;

packages/sdks/express/package.json

@@ -18,6 +18,6 @@
     "@types/express": "^5.0.3",
     "@types/request-ip": "^0.0.41",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/sdks/nextjs/package.json

@@ -16,8 +16,8 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/react": "^18.2.20",
+    "@types/react": "catalog:",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/sdks/react-native/package.json

@@ -11,9 +11,9 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   },
   "peerDependencies": {
     "expo-application": "5 - 6",

packages/sdks/sdk/package.json

@@ -9,8 +9,8 @@
   "dependencies": {},
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/sdks/web/package.json

@@ -11,8 +11,8 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "tsup": "^7.2.0",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/sdks/web/src/types.d.ts

@@ -1,7 +1,5 @@
+import type { TrackProperties } from '@openpanel/sdk';
 import type { OpenPanel, OpenPanelOptions } from './';
-import type {
-  TrackProperties,
-} from '@openpanel/sdk';
 
 type ExposedMethodsNames =
   | 'track'
@@ -22,7 +20,11 @@ export type OpenPanelMethodNames = ExposedMethodsNames | 'init' | 'screenView';
 export type OpenPanelMethods =
   | ExposedMethods
   | ['init', OpenPanelOptions]
-  | ['screenView', string | TrackProperties | undefined, TrackProperties | undefined];
+  | [
+      'screenView',
+      string | TrackProperties | undefined,
+      TrackProperties | undefined,
+    ];
 
 declare global {
   interface Window {

packages/trpc/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/trpc",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -15,11 +16,9 @@
     "@openpanel/payments": "workspace:^",
     "@openpanel/redis": "workspace:*",
     "@openpanel/validation": "workspace:*",
-    "@seventy-seven/sdk": "0.0.0-beta.2",
     "@trpc-limiter/redis": "^0.0.2",
-    "@trpc/server": "^10.45.2",
-    "@trpc/client": "^10.45.2",
-    "bcrypt": "^5.1.1",
+    "@trpc/client": "^11.6.0",
+    "@trpc/server": "^11.6.0",
     "date-fns": "^3.3.1",
     "mathjs": "^12.3.2",
     "prisma-error-enum": "^0.1.3",
@@ -32,11 +31,10 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/bcrypt": "^5.0.2",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "@types/ramda": "^0.29.6",
     "@types/sqlstring": "^2.3.2",
     "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }

packages/trpc/src/root.ts

@@ -1,5 +1,6 @@
 import { authRouter } from './routers/auth';
 import { chartRouter } from './routers/chart';
+import { chatRouter } from './routers/chat';
 import { clientRouter } from './routers/client';
 import { dashboardRouter } from './routers/dashboard';
 import { eventRouter } from './routers/event';
@@ -10,11 +11,12 @@ import { organizationRouter } from './routers/organization';
 import { overviewRouter } from './routers/overview';
 import { profileRouter } from './routers/profile';
 import { projectRouter } from './routers/project';
+import { realtimeRouter } from './routers/realtime';
 import { referenceRouter } from './routers/reference';
 import { reportRouter } from './routers/report';
+import { sessionRouter } from './routers/session';
 import { shareRouter } from './routers/share';
 import { subscriptionRouter } from './routers/subscription';
-import { ticketRouter } from './routers/ticket';
 import { userRouter } from './routers/user';
 import { createTRPCRouter } from './trpc';
 /**
@@ -32,15 +34,17 @@ export const appRouter = createTRPCRouter({
   client: clientRouter,
   event: eventRouter,
   profile: profileRouter,
+  session: sessionRouter,
   share: shareRouter,
   onboarding: onboardingRouter,
   reference: referenceRouter,
-  ticket: ticketRouter,
   notification: notificationRouter,
   integration: integrationRouter,
   auth: authRouter,
   subscription: subscriptionRouter,
   overview: overviewRouter,
+  realtime: realtimeRouter,
+  chat: chatRouter,
 });
 
 // export type definition of API

packages/trpc/src/routers/auth.ts

@@ -9,6 +9,7 @@ import {
   hashPassword,
   invalidateSession,
   setSessionTokenCookie,
+  validateSessionToken,
   verifyPasswordHash,
 } from '@openpanel/auth';
 import { generateSecureId } from '@openpanel/common/server/id';
@@ -26,7 +27,6 @@ import {
   zSignInShare,
   zSignUpEmail,
 } from '@openpanel/validation';
-import * as bcrypt from 'bcrypt';
 import { z } from 'zod';
 import { TRPCAccessError, TRPCNotFoundError } from '../errors';
 import {
@@ -216,21 +216,17 @@ export const authRouter = createTRPCRouter({
             throw TRPCAccessError('Incorrect email or password');
           }
         } else {
-          const validPassword = await bcrypt.compare(
-            password,
-            user.account.password ?? '',
+          throw TRPCAccessError(
+            'Reset your password, old password has expired',
           );
-
-          if (!validPassword) {
-            throw TRPCAccessError('Incorrect email or password');
-          }
         }
       }
 
       const token = generateSessionToken();
       const session = await createSession(token, user.id);
-
+      console.log('session', session);
       setSessionTokenCookie(ctx.setCookie, token, session.expiresAt);
+      console.log('ctx.setCookie', ctx.setCookie);
       return {
         type: 'email',
       };
@@ -318,7 +314,7 @@ export const authRouter = createTRPCRouter({
       await sendEmail('reset-password', {
         to: input.email,
         data: {
-          url: `${process.env.NEXT_PUBLIC_DASHBOARD_URL}/reset-password?token=${token}`,
+          url: `${process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL}/reset-password?token=${token}`,
         },
       });
 
@@ -328,6 +324,26 @@ export const authRouter = createTRPCRouter({
     return ctx.session;
   }),
 
+  extendSession: publicProcedure.mutation(async ({ ctx }) => {
+    if (!ctx.session.session || !ctx.cookies.session) {
+      return { extended: false };
+    }
+
+    const token = ctx.cookies.session;
+    const session = await validateSessionToken(token);
+
+    if (session.session) {
+      // Re-set the cookie with updated expiration
+      setSessionTokenCookie(ctx.setCookie, token, session.session.expiresAt);
+      return {
+        extended: true,
+        expiresAt: session.session.expiresAt,
+      };
+    }
+
+    return { extended: false };
+  }),
+
   signInShare: publicProcedure
     .use(
       rateLimitMiddleware({

packages/trpc/src/routers/chart.helpers.ts

@@ -1,6 +1,6 @@
 import * as mathjs from 'mathjs';
 import { last, pluck, reverse, uniq } from 'ramda';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 
 import type { ISerieDataItem } from '@openpanel/common';
 import {
@@ -162,11 +162,11 @@ export async function getFunnelData({
   const funnels = payload.events.map((event) => {
     const { sb, getWhere } = createSqlBuilder();
     sb.where = getEventFiltersWhereClause(event.filters);
-    sb.where.name = `name = ${escape(event.name)}`;
+    sb.where.name = `name = ${sqlstring.escape(event.name)}`;
     return getWhere().replace('WHERE ', '');
   });
 
-  const commonWhere = `project_id = ${escape(projectId)} AND 
+  const commonWhere = `project_id = ${sqlstring.escape(projectId)} AND 
     created_at >= '${formatClickhouseDate(startDate)}' AND 
     created_at <= '${formatClickhouseDate(endDate)}'`;
 
@@ -177,7 +177,7 @@ export async function getFunnelData({
   ${funnelGroup[0] === 'session_id' ? '' : `LEFT JOIN (SELECT profile_id, id FROM sessions WHERE ${commonWhere}) AS s ON s.id = e.session_id`}
   WHERE 
     ${commonWhere} AND
-    name IN (${payload.events.map((event) => escape(event.name)).join(', ')})
+    name IN (${payload.events.map((event) => sqlstring.escape(event.name)).join(', ')})
   GROUP BY ${funnelGroup[0]}`;
 
   const sql = `SELECT level, count() AS count FROM (${innerSql}) WHERE level != 0 GROUP BY level ORDER BY level DESC`;

packages/trpc/src/routers/chart.ts

@@ -1,5 +1,5 @@
 import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 import { z } from 'zod';
 
 import {
@@ -52,6 +52,80 @@ function utc(date: string | Date) {
 const cacher = cacheMiddleware(60);
 
 export const chartRouter = createTRPCRouter({
+  projectCard: protectedProcedure
+    .use(cacheMiddleware(60 * 5))
+    .input(
+      z.object({
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input: { projectId } }) => {
+      const chartPromise = chQuery<{ value: number; date: Date }>(
+        `SELECT
+            uniqHLL12(profile_id) as value,
+            toStartOfDay(created_at) as date
+        FROM ${TABLE_NAMES.sessions}
+        WHERE 
+            sign = 1 AND 
+            project_id = ${sqlstring.escape(projectId)} AND 
+            created_at >= now() - interval '3 month'
+        GROUP BY date
+        ORDER BY date ASC
+        WITH FILL FROM toStartOfDay(now() - interval '1 month') 
+        TO toStartOfDay(now()) 
+        STEP INTERVAL 1 day
+      `,
+      );
+
+      const metricsPromise = clix(ch)
+        .select<{
+          months_3: number;
+          months_3_prev: number;
+          month: number;
+          day: number;
+          day_prev: number;
+        }>([
+          'uniqHLL12(if(created_at >= (now() - toIntervalMonth(3)), profile_id, null)) AS months_3',
+          'uniqHLL12(if(created_at >= (now() - toIntervalMonth(6)) AND created_at < (now() - toIntervalMonth(3)), profile_id, null)) AS months_3_prev',
+          'uniqHLL12(if(created_at >= (now() - toIntervalMonth(1)), profile_id, null)) AS month',
+          'uniqHLL12(if(created_at >= (now() - toIntervalDay(1)), profile_id, null)) AS day',
+          'uniqHLL12(if(created_at >= (now() - toIntervalDay(2)) AND created_at < (now() - toIntervalDay(1)), profile_id, null)) AS day_prev',
+        ])
+        .from(TABLE_NAMES.sessions)
+        .where('project_id', '=', projectId)
+        .where('created_at', '>=', clix.exp('now() - toIntervalMonth(6)'))
+        .execute();
+
+      const [chart, [metrics]] = await Promise.all([
+        chartPromise,
+        metricsPromise,
+      ]);
+
+      const change =
+        metrics && metrics.months_3_prev > 0 && metrics.months_3 > 0
+          ? Math.round(
+              ((metrics.months_3 - metrics.months_3_prev) /
+                metrics.months_3_prev) *
+                100,
+            )
+          : null;
+
+      const trend =
+        change === null
+          ? { direction: 'neutral' as const, percentage: null as number | null }
+          : change > 0
+            ? { direction: 'up' as const, percentage: change }
+            : change < 0
+              ? { direction: 'down' as const, percentage: Math.abs(change) }
+              : { direction: 'neutral' as const, percentage: 0 };
+
+      return {
+        chart: chart.map((d) => ({ ...d, date: new Date(d.date) })),
+        metrics,
+        trend,
+      };
+    }),
+
   events: protectedProcedure
     .input(
       z.object({
@@ -61,7 +135,7 @@ export const chartRouter = createTRPCRouter({
     .query(async ({ input: { projectId } }) => {
       const [events, meta] = await Promise.all([
         chQuery<{ name: string; count: number }>(
-          `SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`,
+          `SELECT name, count(name) as count FROM ${TABLE_NAMES.event_names_mv} WHERE project_id = ${sqlstring.escape(projectId)} GROUP BY name ORDER BY count DESC, name ASC`,
         ),
         getEventMetasCached(projectId),
       ]);
@@ -376,9 +450,9 @@ export const chartRouter = createTRPCRouter({
 
       const whereEventNameIs = (event: string[]) => {
         if (event.length === 1) {
-          return `name = ${escape(event[0])}`;
+          return `name = ${sqlstring.escape(event[0])}`;
         }
-        return `name IN (${event.map((e) => escape(e)).join(',')})`;
+        return `name IN (${event.map((e) => sqlstring.escape(e)).join(',')})`;
       };
 
       const cohortQuery = `
@@ -390,7 +464,7 @@ export const chartRouter = createTRPCRouter({
             ${sqlToStartOf}(created_at) AS cohort_interval
           FROM ${TABLE_NAMES.cohort_events_mv}
           WHERE ${whereEventNameIs(firstEvent)}
-            AND project_id = ${escape(projectId)}
+            AND project_id = ${sqlstring.escape(projectId)}
             AND created_at BETWEEN toDate('${utc(dates.startDate)}') AND toDate('${utc(dates.endDate)}')
         ),
         last_event AS
@@ -401,7 +475,7 @@ export const chartRouter = createTRPCRouter({
                 toDate(created_at) AS event_date
             FROM cohort_events_mv
             WHERE ${whereEventNameIs(secondEvent)}
-            AND project_id = ${escape(projectId)}
+            AND project_id = ${sqlstring.escape(projectId)}
             AND created_at BETWEEN toDate('${utc(dates.startDate)}') AND toDate('${utc(dates.endDate)}') + INTERVAL ${diffInterval} ${sqlInterval}
         ),
         retention_matrix AS

packages/trpc/src/routers/chat.ts

@@ -0,0 +1,20 @@
+import { z } from 'zod';
+
+import { db } from '@openpanel/db';
+
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export const chatRouter = createTRPCRouter({
+  get: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input }) => {
+      return db.chat.findFirst({
+        where: {
+          projectId: input.projectId,
+        },
+        orderBy: {
+          createdAt: 'desc',
+        },
+      });
+    }),
+});

packages/trpc/src/routers/client.ts

@@ -10,6 +10,19 @@ import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 export const clientRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input }) => {
+      return db.client.findMany({
+        where: {
+          projectId: input.projectId,
+        },
+      });
+    }),
   update: protectedProcedure
     .input(
       z.object({

packages/trpc/src/routers/dashboard.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 
 import {
   db,
+  getDashboardById,
   getDashboardsByProjectId,
   getId,
   getProjectById,
@@ -23,6 +24,31 @@ export const dashboardRouter = createTRPCRouter({
     .query(({ input }) => {
       return getDashboardsByProjectId(input.projectId);
     }),
+  byId: protectedProcedure
+    .input(
+      z.object({
+        id: z.string(),
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      const access = await getProjectAccess({
+        projectId: input.projectId,
+        userId: ctx.session.userId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      const dashboard = await getDashboardById(input.id, input.projectId);
+
+      if (!dashboard) {
+        throw TRPCNotFoundError('Dashboard not found');
+      }
+
+      return dashboard;
+    }),
   create: protectedProcedure
     .input(
       z.object({

packages/trpc/src/routers/event.ts

@@ -1,5 +1,5 @@
 import { TRPCError } from '@trpc/server';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 import { z } from 'zod';
 
 import {
@@ -117,6 +117,7 @@ export const eventRouter = createTRPCRouter({
       z.object({
         projectId: z.string(),
         profileId: z.string().optional(),
+        sessionId: z.string().optional(),
         cursor: z.string().optional(),
         filters: z.array(zChartEventFilter).default([]),
         startDate: z.date().optional(),
@@ -130,10 +131,15 @@ export const eventRouter = createTRPCRouter({
         take: 50,
         cursor: input.cursor ? new Date(input.cursor) : undefined,
         select: {
+          profile: true,
           properties: true,
           sessionId: true,
           deviceId: true,
           profileId: true,
+          referrerName: true,
+          referrerType: true,
+          referrer: true,
+          origin: true,
         },
       });
 
@@ -159,7 +165,7 @@ export const eventRouter = createTRPCRouter({
       const lastItem = items[items.length - 1];
 
       return {
-        items,
+        data: items,
         meta: {
           next:
             items.length === 50 && lastItem
@@ -168,39 +174,80 @@ export const eventRouter = createTRPCRouter({
         },
       };
     }),
+  conversionNames: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input: { projectId } }) => {
+      return getConversionEventNames(projectId);
+    }),
   conversions: protectedProcedure
     .input(
       z.object({
         projectId: z.string(),
         cursor: z.string().optional(),
+        startDate: z.date().optional(),
+        endDate: z.date().optional(),
       }),
     )
-    .query(async ({ input: { projectId, cursor } }) => {
-      const conversions = await getConversionEventNames(projectId);
+    .query(async ({ input }) => {
+      const conversions = await getConversionEventNames(input.projectId);
 
       if (conversions.length === 0) {
         return {
-          items: [],
+          data: [],
           meta: {
             next: null,
           },
         };
       }
 
-      const items = await getEvents(
-        `SELECT * FROM ${TABLE_NAMES.events} WHERE ${cursor ? `created_at <= '${formatClickhouseDate(cursor)}' AND` : ''} project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY toDate(created_at) DESC, created_at DESC LIMIT 50;`,
-        {
+      const items = await getEventList({
+        ...input,
+        take: 50,
+        cursor: input.cursor ? new Date(input.cursor) : undefined,
+        select: {
           profile: true,
-          meta: true,
+          properties: true,
+          sessionId: true,
+          deviceId: true,
+          profileId: true,
+          referrerName: true,
+          referrerType: true,
+          referrer: true,
+          origin: true,
         },
-      );
+        custom: (sb) => {
+          sb.where.name = `name IN (${conversions.map((event) => sqlstring.escape(event.name)).join(',')})`;
+        },
+      });
+
+      // Hacky join to get profile for entire session
+      // TODO: Replace this with a join on the session table
+      const map = new Map<string, IServiceProfile>(); // sessionId -> profileId
+      for (const item of items) {
+        if (item.sessionId && item.profile?.isExternal === true) {
+          map.set(item.sessionId, item.profile);
+        }
+      }
+
+      for (const item of items) {
+        const profile = map.get(item.sessionId);
+        if (profile && (item.profile?.isExternal === false || !item.profile)) {
+          item.profile = clone(profile);
+          if (item?.profile?.firstName) {
+            item.profile.firstName = `* ${item.profile.firstName}`;
+          }
+        }
+      }
 
       const lastItem = items[items.length - 1];
 
       return {
-        items,
+        data: items,
         meta: {
-          next: lastItem ? lastItem.createdAt.toISOString() : null,
+          next:
+            items.length === 50 && lastItem
+              ? lastItem.createdAt.toISOString()
+              : null,
         },
       };
     }),
@@ -243,12 +290,12 @@ export const eventRouter = createTRPCRouter({
           path: string;
           created_at: string;
         }>(
-          `SELECT * FROM ${TABLE_NAMES.events_bots} WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit} OFFSET ${(cursor ?? 0) * limit}`,
+          `SELECT * FROM ${TABLE_NAMES.events_bots} WHERE project_id = ${sqlstring.escape(projectId)} ORDER BY created_at DESC LIMIT ${limit} OFFSET ${(cursor ?? 0) * limit}`,
         ),
         chQuery<{
           count: number;
         }>(
-          `SELECT count(*) as count FROM ${TABLE_NAMES.events_bots} WHERE project_id = ${escape(projectId)}`,
+          `SELECT count(*) as count FROM ${TABLE_NAMES.events_bots} WHERE project_id = ${sqlstring.escape(projectId)}`,
         ),
       ]);
 
@@ -303,7 +350,7 @@ export const eventRouter = createTRPCRouter({
     )
     .query(async ({ input }) => {
       const res = await chQuery<{ origin: string }>(
-        `SELECT DISTINCT origin FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(
+        `SELECT DISTINCT origin FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(
           input.projectId,
         )} AND origin IS NOT NULL AND origin != '' AND toDate(created_at) > now() - INTERVAL 30 DAY ORDER BY origin ASC`,
       );

packages/trpc/src/routers/notification.ts

@@ -37,7 +37,7 @@ export const notificationRouter = createTRPCRouter({
             },
           },
         },
-        take: 100,
+        take: 5000,
       });
     }),
   rules: protectedProcedure

packages/trpc/src/routers/onboarding.ts

@@ -33,10 +33,7 @@ async function createOrGetOrganization(
       },
     });
 
-    if (
-      process.env.NEXT_PUBLIC_SELF_HOSTED !== 'true' &&
-      !process.env.SELF_HOSTED
-    ) {
+    if (process.env.VITE_SELF_HOSTED !== 'true' && !process.env.SELF_HOSTED) {
       await addTrialEndingSoonJob(
         organization.id,
         1000 * 60 * 60 * 24 * TRIAL_DURATION_IN_DAYS * 0.9,
@@ -64,7 +61,6 @@ export const onboardingRouter = createTRPCRouter({
     if (members.length > 0) {
       return {
         canSkip: true,
-        url: `/${members[0]?.organizationId}`,
       };
     }
 
@@ -77,11 +73,10 @@ export const onboardingRouter = createTRPCRouter({
     if (projectAccess.length > 0) {
       return {
         canSkip: true,
-        url: `/${projectAccess[0]?.organizationId}/${projectAccess[0]?.projectId}`,
       };
     }
 
-    return { canSkip: false, url: null };
+    return { canSkip: false };
   }),
   project: protectedProcedure
     .input(zOnboardingProject)

packages/trpc/src/routers/organization.ts

@@ -1,6 +1,14 @@
 import { z } from 'zod';
 
-import { connectUserToOrganization, db } from '@openpanel/db';
+import {
+  connectUserToOrganization,
+  db,
+  getInviteById,
+  getInvites,
+  getMembers,
+  getOrganizationById,
+  getOrganizations,
+} from '@openpanel/db';
 import { zEditOrganization, zInviteUser } from '@openpanel/validation';
 
 import { generateSecureId } from '@openpanel/common/server/id';
@@ -8,9 +16,24 @@ import { sendEmail } from '@openpanel/email';
 import { addDays } from 'date-fns';
 import { getOrganizationAccess } from '../access';
 import { TRPCAccessError, TRPCBadRequestError } from '../errors';
-import { createTRPCRouter, protectedProcedure } from '../trpc';
+import {
+  createTRPCRouter,
+  protectedProcedure,
+  publicProcedure,
+  rateLimitMiddleware,
+} from '../trpc';
 
 export const organizationRouter = createTRPCRouter({
+  get: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .query(async ({ input }) => {
+      return getOrganizationById(input.organizationId);
+    }),
+
+  list: protectedProcedure.query(async ({ ctx }) => {
+    return getOrganizations(ctx.session.userId);
+  }),
+
   update: protectedProcedure
     .input(zEditOrganization)
     .mutation(async ({ input, ctx }) => {
@@ -116,7 +139,7 @@ export const organizationRouter = createTRPCRouter({
       await sendEmail('invite', {
         to: email,
         data: {
-          url: `${process.env.NEXT_PUBLIC_DASHBOARD_URL}/onboarding?inviteId=${invite.id}`,
+          url: `${process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL}/onboarding?inviteId=${invite.id}`,
           organizationName: invite.organization.name,
         },
       });
@@ -240,4 +263,31 @@ export const organizationRouter = createTRPCRouter({
         }),
       ]);
     }),
+
+  members: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .query(async ({ input }) => {
+      return getMembers(input.organizationId);
+    }),
+
+  invitations: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .query(async ({ input }) => {
+      return getInvites(input.organizationId);
+    }),
+
+  getInvite: publicProcedure
+    .use(
+      rateLimitMiddleware({
+        max: 5,
+        windowMs: 30_000,
+      }),
+    )
+    .input(z.object({ inviteId: z.string().optional() }))
+    .query(async ({ input }) => {
+      if (!input.inviteId) {
+        throw TRPCBadRequestError('Invite ID is required');
+      }
+      return getInviteById(input.inviteId);
+    }),
 });

packages/trpc/src/routers/overview.ts

@@ -1,4 +1,5 @@
 import {
+  eventBuffer,
   getChartPrevStartEndDate,
   getChartStartEndDate,
   getOrganizationSubscriptionChartEndDate,
@@ -76,6 +77,11 @@ function getCurrentAndPrevious<
 }
 
 export const overviewRouter = createTRPCRouter({
+  liveVisitors: publicProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input }) => {
+      return eventBuffer.getActiveVisitorCount(input.projectId);
+    }),
   stats: publicProcedure
     .input(
       zGetMetricsInput.omit({ startDate: true, endDate: true }).extend({

packages/trpc/src/routers/profile.ts

@@ -1,23 +1,62 @@
 import { flatten, map, pipe, prop, sort, uniq } from 'ramda';
-import { escape } from 'sqlstring';
+import sqlstring from 'sqlstring';
 import { z } from 'zod';
 
 import {
   TABLE_NAMES,
   chQuery,
   createSqlBuilder,
+  getProfileByIdCached,
   getProfileList,
+  getProfileListCount,
+  getProfileMetrics,
   getProfiles,
 } from '@openpanel/db';
 
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 export const profileRouter = createTRPCRouter({
+  byId: protectedProcedure
+    .input(z.object({ profileId: z.string(), projectId: z.string() }))
+    .query(async ({ input: { profileId, projectId } }) => {
+      return getProfileByIdCached(profileId, projectId);
+    }),
+
+  metrics: protectedProcedure
+    .input(z.object({ profileId: z.string(), projectId: z.string() }))
+    .query(async ({ input: { profileId, projectId } }) => {
+      return getProfileMetrics(profileId, projectId);
+    }),
+
+  activity: protectedProcedure
+    .input(z.object({ profileId: z.string(), projectId: z.string() }))
+    .query(async ({ input: { profileId, projectId } }) => {
+      return chQuery<{ count: number; date: string }>(
+        `SELECT count(*) as count, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} and profile_id = ${sqlstring.escape(profileId)} GROUP BY date ORDER BY date DESC`,
+      );
+    }),
+
+  mostEvents: protectedProcedure
+    .input(z.object({ profileId: z.string(), projectId: z.string() }))
+    .query(async ({ input: { profileId, projectId } }) => {
+      return chQuery<{ count: number; name: string }>(
+        `SELECT count(*) as count, name FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${sqlstring.escape(projectId)} and profile_id = ${sqlstring.escape(profileId)} GROUP BY name ORDER BY count DESC`,
+      );
+    }),
+
+  popularRoutes: protectedProcedure
+    .input(z.object({ profileId: z.string(), projectId: z.string() }))
+    .query(async ({ input: { profileId, projectId } }) => {
+      return chQuery<{ count: number; path: string }>(
+        `SELECT count(*) as count, path FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND project_id = ${sqlstring.escape(projectId)} and profile_id = ${sqlstring.escape(profileId)} GROUP BY path ORDER BY count DESC LIMIT 10`,
+      );
+    }),
+
   properties: protectedProcedure
     .input(z.object({ projectId: z.string() }))
     .query(async ({ input: { projectId } }) => {
       const events = await chQuery<{ keys: string[] }>(
-        `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.profiles} where project_id = ${escape(projectId)};`,
+        `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.profiles} where project_id = ${sqlstring.escape(projectId)};`,
       );
 
       const properties = events
@@ -41,11 +80,21 @@ export const profileRouter = createTRPCRouter({
         cursor: z.number().optional(),
         take: z.number().default(50),
         search: z.string().optional(),
-        // filters: z.array(zChartEventFilter).default([]),
+        isExternal: z.boolean().optional(),
       }),
     )
-    .query(async ({ input: { projectId, cursor, take, search } }) => {
-      return getProfileList({ projectId, cursor, take, search });
+    .query(async ({ input }) => {
+      const [data, count] = await Promise.all([
+        getProfileList(input),
+        getProfileListCount(input),
+      ]);
+      return {
+        data,
+        meta: {
+          count,
+          pageCount: input.take,
+        },
+      };
     }),
 
   powerUsers: protectedProcedure
@@ -54,28 +103,42 @@ export const profileRouter = createTRPCRouter({
         projectId: z.string(),
         cursor: z.number().optional(),
         take: z.number().default(50),
-        // filters: z.array(zChartEventFilter).default([]),
       }),
     )
     .query(async ({ input: { projectId, cursor, take } }) => {
       const res = await chQuery<{ profile_id: string; count: number }>(
-        `SELECT profile_id, count(*) as count from ${TABLE_NAMES.events} where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT ${take} ${cursor ? `OFFSET ${cursor * take}` : ''}`,
+        `
+        SELECT profile_id, count(*) as count 
+        FROM ${TABLE_NAMES.events} 
+        WHERE 
+          profile_id != '' 
+          AND project_id = ${sqlstring.escape(projectId)} 
+          GROUP BY profile_id 
+          ORDER BY count() DESC 
+          LIMIT ${take} ${cursor ? `OFFSET ${cursor * take}` : ''}`,
       );
       const profiles = await getProfiles(
         res.map((r) => r.profile_id),
         projectId,
       );
-      return (
-        res
-          .map((item) => {
-            return {
-              count: item.count,
-              ...(profiles.find((p) => p.id === item.profile_id)! ?? {}),
-            };
-          })
-          // Make sure we return actual profiles
-          .filter((item) => item.id)
-      );
+
+      const data = res
+        .map((item) => {
+          return {
+            count: item.count,
+            ...(profiles.find((p) => p.id === item.profile_id)! ?? {}),
+          };
+        })
+        // Make sure we return actual profiles
+        .filter((item) => item.id);
+
+      return {
+        data,
+        meta: {
+          count: data.length,
+          pageCount: take,
+        },
+      };
     }),
 
   values: protectedProcedure
@@ -88,9 +151,9 @@ export const profileRouter = createTRPCRouter({
     .query(async ({ input: { property, projectId } }) => {
       const { sb, getSql } = createSqlBuilder();
       sb.from = TABLE_NAMES.profiles;
-      sb.where.project_id = `project_id = ${escape(projectId)}`;
+      sb.where.project_id = `project_id = ${sqlstring.escape(projectId)}`;
       if (property.startsWith('properties.')) {
-        sb.select.values = `distinct arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
+        sb.select.values = `distinct arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${sqlstring.escape(
           property.replace(/^properties\./, '').replace('.*.', '.%.'),
         )}))) as values`;
       } else {

packages/trpc/src/routers/project.ts

@@ -6,19 +6,38 @@ import { hashPassword } from '@openpanel/common/server';
 import {
   type Prisma,
   db,
-  getClientById,
   getClientByIdCached,
   getId,
   getProjectByIdCached,
+  getProjectWithClients,
   getProjectsByOrganizationId,
 } from '@openpanel/db';
 import { zOnboardingProject, zProject } from '@openpanel/validation';
-import { addDays, addHours } from 'date-fns';
+import { addHours } from 'date-fns';
 import { getProjectAccess } from '../access';
 import { TRPCAccessError, TRPCBadRequestError } from '../errors';
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 export const projectRouter = createTRPCRouter({
+  getProjectWithClients: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input: { projectId }, ctx }) => {
+      const access = await getProjectAccess({
+        userId: ctx.session.userId,
+        projectId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      return getProjectWithClients(projectId);
+    }),
+
   list: protectedProcedure
     .input(
       z.object({

packages/trpc/src/routers/realtime.ts

@@ -0,0 +1,143 @@
+import { z } from 'zod';
+
+import {
+  type EventMeta,
+  TABLE_NAMES,
+  ch,
+  chQuery,
+  clix,
+  db,
+  formatClickhouseDate,
+  getEventList,
+} from '@openpanel/db';
+
+import { subMinutes } from 'date-fns';
+import sqlstring from 'sqlstring';
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export const realtimeRouter = createTRPCRouter({
+  coordinates: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input }) => {
+      const res = await chQuery<{
+        city: string;
+        country: string;
+        long: number;
+        lat: number;
+      }>(
+        `SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`,
+      );
+
+      return res;
+    }),
+  activeSessions: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input }) => {
+      return getEventList({
+        projectId: input.projectId,
+        take: 30,
+        select: {
+          name: true,
+          path: true,
+          origin: true,
+          referrer: true,
+          referrerName: true,
+          referrerType: true,
+          country: true,
+          device: true,
+          os: true,
+          browser: true,
+          createdAt: true,
+          profile: true,
+          meta: true,
+        },
+      });
+    }),
+  paths: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input }) => {
+      const res = await clix(ch)
+        .select<{
+          origin: string;
+          path: string;
+          count: number;
+          avg_duration: number;
+        }>([
+          'origin',
+          'path',
+          'COUNT(*) as count',
+          'round(avg(duration)/1000, 2) as avg_duration',
+        ])
+        .from(TABLE_NAMES.events)
+        .where('project_id', '=', input.projectId)
+        .where('path', '!=', '')
+        .where(
+          'created_at',
+          '>=',
+          formatClickhouseDate(subMinutes(new Date(), 30)),
+        )
+        .groupBy(['path', 'origin'])
+        .orderBy('count', 'DESC')
+        .limit(100)
+        .execute();
+
+      return res;
+    }),
+  referrals: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input }) => {
+      const res = await clix(ch)
+        .select<{
+          referrer_name: string;
+          count: number;
+          avg_duration: number;
+        }>([
+          'referrer_name',
+          'COUNT(*) as count',
+          'round(avg(duration)/1000, 2) as avg_duration',
+        ])
+        .from(TABLE_NAMES.events)
+        .where('project_id', '=', input.projectId)
+        .where('referrer_name', 'IS NOT NULL')
+        .where(
+          'created_at',
+          '>=',
+          formatClickhouseDate(subMinutes(new Date(), 30)),
+        )
+        .groupBy(['referrer_name'])
+        .orderBy('count', 'DESC')
+        .limit(100)
+        .execute();
+
+      return res;
+    }),
+  geo: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input }) => {
+      const res = await clix(ch)
+        .select<{
+          country: string;
+          city: string;
+          count: number;
+          avg_duration: number;
+        }>([
+          'country',
+          'city',
+          'COUNT(*) as count',
+          'round(avg(duration)/1000, 2) as avg_duration',
+        ])
+        .from(TABLE_NAMES.events)
+        .where('project_id', '=', input.projectId)
+        .where(
+          'created_at',
+          '>=',
+          formatClickhouseDate(subMinutes(new Date(), 30)),
+        )
+        .groupBy(['country', 'city'])
+        .orderBy('count', 'DESC')
+        .limit(100)
+        .execute();
+
+      return res;
+    }),
+});

packages/trpc/src/routers/reference.ts

@@ -1,11 +1,6 @@
 import { z } from 'zod';
 
-import {
-  db,
-  getChartStartEndDate,
-  getReferences,
-  getSettingsForProject,
-} from '@openpanel/db';
+import { db, getChartStartEndDate, getSettingsForProject } from '@openpanel/db';
 import { zCreateReference, zRange } from '@openpanel/validation';
 
 import { getProjectAccess } from '../access';
@@ -13,6 +8,32 @@ import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
 
 export const referenceRouter = createTRPCRouter({
+  getReferences: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        cursor: z.number().optional(),
+      }),
+    )
+    .query(async ({ input: { projectId, cursor }, ctx }) => {
+      const access = await getProjectAccess({
+        userId: ctx.session.userId,
+        projectId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      return db.reference.findMany({
+        where: {
+          projectId,
+        },
+        take: 50,
+        skip: cursor ? cursor * 50 : 0,
+      });
+    }),
+
   create: protectedProcedure
     .input(zCreateReference)
     .mutation(
@@ -27,6 +48,38 @@ export const referenceRouter = createTRPCRouter({
         });
       },
     ),
+  update: protectedProcedure
+    .input(
+      z.object({
+        id: z.string(),
+        title: z.string(),
+        description: z.string().nullish(),
+        datetime: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const existing = await db.reference.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+
+      const access = await getProjectAccess({
+        userId: ctx.session.userId,
+        projectId: existing.projectId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      return db.reference.update({
+        where: { id: input.id },
+        data: {
+          title: input.title,
+          description: input.description ?? null,
+          date: new Date(input.datetime),
+        },
+      });
+    }),
   delete: protectedProcedure
     .input(z.object({ id: z.string() }))
     .mutation(async ({ input: { id }, ctx }) => {
@@ -63,7 +116,7 @@ export const referenceRouter = createTRPCRouter({
     .query(async ({ input: { projectId, ...input } }) => {
       const { timezone } = await getSettingsForProject(projectId);
       const { startDate, endDate } = getChartStartEndDate(input, timezone);
-      return getReferences({
+      return db.reference.findMany({
         where: {
           projectId,
           date: {

packages/trpc/src/routers/report.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 
-import { db } from '@openpanel/db';
+import { db, getReportById, getReportsByDashboardId } from '@openpanel/db';
 import { zReportInput } from '@openpanel/validation';
 
 import { getProjectAccess } from '../access';
@@ -8,6 +8,16 @@ import { TRPCAccessError } from '../errors';
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 export const reportRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(
+      z.object({
+        dashboardId: z.string(),
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input: { dashboardId, projectId }, ctx }) => {
+      return getReportsByDashboardId(dashboardId);
+    }),
   create: protectedProcedure
     .input(
       z.object({
@@ -125,4 +135,127 @@ export const reportRouter = createTRPCRouter({
         },
       });
     }),
+  get: protectedProcedure
+    .input(
+      z.object({
+        reportId: z.string(),
+      }),
+    )
+    .query(async ({ input: { reportId }, ctx }) => {
+      return getReportById(reportId);
+    }),
+  updateLayout: protectedProcedure
+    .input(
+      z.object({
+        reportId: z.string(),
+        layout: z.object({
+          x: z.number(),
+          y: z.number(),
+          w: z.number(),
+          h: z.number(),
+          minW: z.number().optional(),
+          minH: z.number().optional(),
+          maxW: z.number().optional(),
+          maxH: z.number().optional(),
+        }),
+      }),
+    )
+    .mutation(async ({ input: { reportId, layout }, ctx }) => {
+      const report = await db.report.findUniqueOrThrow({
+        where: {
+          id: reportId,
+        },
+      });
+
+      const access = await getProjectAccess({
+        userId: ctx.session.userId,
+        projectId: report.projectId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      // Upsert the layout (create if doesn't exist, update if it does)
+      return db.reportLayout.upsert({
+        where: {
+          reportId: reportId,
+        },
+        create: {
+          reportId: reportId,
+          x: layout.x,
+          y: layout.y,
+          w: layout.w,
+          h: layout.h,
+          minW: layout.minW,
+          minH: layout.minH,
+          maxW: layout.maxW,
+          maxH: layout.maxH,
+        },
+        update: {
+          x: layout.x,
+          y: layout.y,
+          w: layout.w,
+          h: layout.h,
+          minW: layout.minW,
+          minH: layout.minH,
+          maxW: layout.maxW,
+          maxH: layout.maxH,
+        },
+      });
+    }),
+  getLayouts: protectedProcedure
+    .input(
+      z.object({
+        dashboardId: z.string(),
+        projectId: z.string(),
+      }),
+    )
+    .query(async ({ input: { dashboardId, projectId }, ctx }) => {
+      const access = await getProjectAccess({
+        userId: ctx.session.userId,
+        projectId: projectId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      return db.reportLayout.findMany({
+        where: {
+          report: {
+            dashboardId: dashboardId,
+          },
+        },
+        include: {
+          report: true,
+        },
+      });
+    }),
+  resetLayout: protectedProcedure
+    .input(
+      z.object({
+        dashboardId: z.string(),
+        projectId: z.string(),
+      }),
+    )
+    .mutation(async ({ input: { dashboardId, projectId }, ctx }) => {
+      const access = await getProjectAccess({
+        userId: ctx.session.userId,
+        projectId: projectId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      // Delete all layout data for reports in this dashboard
+      return db.reportLayout.deleteMany({
+        where: {
+          report: {
+            dashboardId: dashboardId,
+          },
+        },
+      });
+    }),
 });

packages/trpc/src/routers/session.ts

@@ -0,0 +1,64 @@
+import { z } from 'zod';
+
+import { getSessionList, sessionService } from '@openpanel/db';
+import { zChartEventFilter } from '@openpanel/validation';
+
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export function encodeCursor(cursor: {
+  createdAt: string;
+  id: string;
+}): string {
+  const json = JSON.stringify(cursor);
+  return Buffer.from(json, 'utf8').toString('base64url'); // URL-safe
+}
+
+export function decodeCursor(
+  encoded: string,
+): { createdAt: string; id: string } | null {
+  try {
+    const json = Buffer.from(encoded, 'base64url').toString('utf8');
+    const obj = JSON.parse(json);
+    if (typeof obj.createdAt === 'string' && typeof obj.id === 'string') {
+      return obj;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+export const sessionRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        profileId: z.string().optional(),
+        cursor: z.string().nullish(),
+        filters: z.array(zChartEventFilter).default([]),
+        startDate: z.date().optional(),
+        endDate: z.date().optional(),
+        search: z.string().optional(),
+        take: z.number().default(50),
+      }),
+    )
+    .query(async ({ input }) => {
+      const cursor = input.cursor ? decodeCursor(input.cursor) : null;
+      const data = await getSessionList({
+        ...input,
+        cursor,
+      });
+      return {
+        data: data.items,
+        meta: {
+          next: data.meta.next ? encodeCursor(data.meta.next) : undefined,
+        },
+      };
+    }),
+
+  byId: protectedProcedure
+    .input(z.object({ sessionId: z.string(), projectId: z.string() }))
+    .query(async ({ input: { sessionId, projectId } }) => {
+      return sessionService.byId(sessionId, projectId);
+    }),
+});

packages/trpc/src/routers/share.ts

@@ -4,12 +4,59 @@ import { db } from '@openpanel/db';
 import { zShareOverview } from '@openpanel/validation';
 
 import { hashPassword } from '@openpanel/auth';
+import { z } from 'zod';
+import { TRPCNotFoundError } from '../errors';
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 const uid = new ShortUniqueId({ length: 6 });
 
 export const shareRouter = createTRPCRouter({
-  shareOverview: protectedProcedure
+  overview: protectedProcedure
+    .input(
+      z
+        .object({
+          projectId: z.string(),
+        })
+        .or(
+          z.object({
+            shareId: z.string(),
+          }),
+        ),
+    )
+    .query(async ({ input, ctx }) => {
+      const share = await db.shareOverview.findUnique({
+        include: {
+          organization: {
+            select: {
+              name: true,
+            },
+          },
+          project: {
+            select: {
+              name: true,
+            },
+          },
+        },
+        where:
+          'projectId' in input
+            ? {
+                projectId: input.projectId,
+              }
+            : {
+                id: input.shareId,
+              },
+      });
+
+      if (!share) {
+        throw TRPCNotFoundError('Share not found');
+      }
+
+      return {
+        ...share,
+        hasAccess: !!ctx.cookies[`shared-overview-${share?.id}`],
+      };
+    }),
+  createOverview: protectedProcedure
     .input(zShareOverview)
     .mutation(async ({ input }) => {
       const passwordHash = input.password

packages/trpc/src/routers/subscription.ts

@@ -75,7 +75,7 @@ export const subscriptionRouter = createTRPCRouter({
       }
 
       const checkout = await createCheckout({
-        priceId: input.productPriceId,
+        productId: input.productId,
         organizationId: input.organizationId,
         projectId: input.projectId ?? undefined,
         user,

packages/trpc/src/routers/ticket.ts

@@ -1,37 +0,0 @@
-import { SeventySevenClient } from '@seventy-seven/sdk';
-import { z } from 'zod';
-
-import { getUserById } from '@openpanel/db';
-
-import { createTRPCRouter, protectedProcedure } from '../trpc';
-
-const API_KEY = process.env.SEVENTY_SEVEN_API_KEY!;
-const client = new SeventySevenClient(API_KEY);
-
-export const ticketRouter = createTRPCRouter({
-  create: protectedProcedure
-    .input(
-      z.object({
-        subject: z.string(),
-        body: z.string(),
-        meta: z.record(z.string(), z.unknown()),
-      }),
-    )
-    .mutation(async ({ input, ctx }) => {
-      if (!API_KEY) {
-        throw new Error('Ticket system not configured');
-      }
-
-      const user = await getUserById(ctx.session.userId);
-
-      return client.createTicket({
-        subject: input.subject,
-        body: input.body,
-        meta: input.meta,
-        senderEmail: user?.email || 'none',
-        senderFullName: user?.firstName
-          ? [user?.firstName, user?.lastName].filter(Boolean).join(' ')
-          : 'none',
-      });
-    }),
-});

packages/trpc/src/routers/user.ts

@@ -5,6 +5,12 @@ import { db } from '@openpanel/db';
 import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
 
 export const userRouter = createTRPCRouter({
+  test: publicProcedure.query(async ({ ctx }) => {
+    const user = await db.user.findFirst();
+    return {
+      user,
+    };
+  }),
   update: protectedProcedure
     .input(
       z.object({

packages/trpc/src/trpc.ts

@@ -4,8 +4,12 @@ import { has } from 'ramda';
 import superjson from 'superjson';
 import { ZodError } from 'zod';
 
-import { COOKIE_OPTIONS, validateSessionToken } from '@openpanel/auth';
-import { getRedisCache } from '@openpanel/redis';
+import {
+  COOKIE_OPTIONS,
+  EMPTY_SESSION,
+  validateSessionToken,
+} from '@openpanel/auth';
+import { getCache, getRedisCache } from '@openpanel/redis';
 import type { ISetCookie } from '@openpanel/validation';
 import {
   createTrpcRedisLimiter,
@@ -31,6 +35,7 @@ export const rateLimitMiddleware = ({
   });
 
 export async function createContext({ req, res }: CreateFastifyContextOptions) {
+  const cookies = (req as any).cookies as Record<string, string | undefined>;
   const setCookie: ISetCookie = (key, value, options) => {
     // @ts-ignore
     res.setCookie(key, value, {
@@ -39,8 +44,17 @@ export async function createContext({ req, res }: CreateFastifyContextOptions) {
     });
   };
 
-  // @ts-ignore
-  const session = await validateSessionToken(req.cookies?.session);
+  const session = cookies?.session
+    ? await getCache(`session:${cookies?.session}`, 1000 * 60 * 5, async () => {
+        return validateSessionToken(cookies.session!);
+      })
+    : EMPTY_SESSION;
+
+  if (process.env.NODE_ENV !== 'production') {
+    await new Promise((res) =>
+      setTimeout(() => res(1), Math.min(Math.random() * 500, 200)),
+    );
+  }
 
   return {
     req,
@@ -49,6 +63,7 @@ export async function createContext({ req, res }: CreateFastifyContextOptions) {
     // we do not get types for `setCookie` from fastify
     // so define it here and be safe in routers
     setCookie,
+    cookies,
   };
 }
 export type Context = Awaited<ReturnType<typeof createContext>>;
@@ -88,7 +103,8 @@ const enforceUserIsAuthed = t.middleware(async ({ ctx, next }) => {
 });
 
 // Only used on protected routes
-const enforceAccess = t.middleware(async ({ ctx, next, rawInput, type }) => {
+const enforceAccess = t.middleware(async ({ ctx, next, type, getRawInput }) => {
+  const rawInput = await getRawInput();
   if (type === 'mutation' && process.env.DEMO_USER_ID) {
     throw new TRPCError({
       code: 'UNAUTHORIZED',
@@ -124,7 +140,8 @@ const enforceAccess = t.middleware(async ({ ctx, next, rawInput, type }) => {
 export const createTRPCRouter = t.router;
 
 const loggerMiddleware = t.middleware(
-  async ({ ctx, next, rawInput, path, input, type }) => {
+  async ({ ctx, next, getRawInput, path, input, type }) => {
+    const rawInput = await getRawInput();
     // Only log mutations
     if (type === 'mutation') {
       ctx.req.log.info('TRPC mutation', {
@@ -153,7 +170,8 @@ const middlewareMarker = 'middlewareMarker' as 'middlewareMarker' & {
 };
 
 export const cacheMiddleware = (cbOrTtl: number | ((input: any) => number)) =>
-  t.middleware(async ({ ctx, next, path, type, rawInput, input }) => {
+  t.middleware(async ({ ctx, next, path, type, getRawInput, input }) => {
+    const rawInput = await getRawInput();
     if (type !== 'query') {
       return next();
     }

packages/validation/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@openpanel/validation",
   "version": "0.0.1",
+  "type": "module",
   "main": "index.ts",
   "scripts": {
     "typecheck": "tsc --noEmit"
@@ -11,8 +12,8 @@
   },
   "devDependencies": {
     "@openpanel/tsconfig": "workspace:*",
-    "@types/node": "20.14.8",
+    "@types/node": "catalog:",
     "prisma": "^5.1.1",
-    "typescript": "^5.2.2"
+    "typescript": "catalog:"
   }
 }