feature(dashboard): refactor overview

fix(lint)

packages/db/code-migrations/3-init-ch.sql

@@ -0,0 +1,167 @@
+CREATE DATABASE IF NOT EXISTS openpanel;
+
+---
+
+CREATE TABLE IF NOT EXISTS self_hosting (
+  `created_at` Date,
+  `domain` String,
+  `count` UInt64
+)
+ENGINE = MergeTree()
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (domain, created_at);
+
+---
+
+CREATE TABLE IF NOT EXISTS events (
+  `id` UUID DEFAULT generateUUIDv4(),
+  `name` LowCardinality(String),
+  `sdk_name` LowCardinality(String),
+  `sdk_version` LowCardinality(String),
+  `device_id` String CODEC(ZSTD(3)),
+  `profile_id` String CODEC(ZSTD(3)),
+  `project_id` String CODEC(ZSTD(3)),
+  `session_id` String CODEC(LZ4),
+  `path` String CODEC(ZSTD(3)),
+  `origin` String CODEC(ZSTD(3)),
+  `referrer` String CODEC(ZSTD(3)),
+  `referrer_name` String CODEC(ZSTD(3)),
+  `referrer_type` LowCardinality(String),
+  `duration` UInt64 CODEC(Delta(4), LZ4),
+  `properties` Map(String, String) CODEC(ZSTD(3)),
+  `created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),
+  `country` LowCardinality(FixedString(2)),
+  `city` String,
+  `region` LowCardinality(String),
+  `longitude` Nullable(Float32) CODEC(Gorilla, LZ4),
+  `latitude` Nullable(Float32) CODEC(Gorilla, LZ4),
+  `os` LowCardinality(String),
+  `os_version` LowCardinality(String),
+  `browser` LowCardinality(String),
+  `browser_version` LowCardinality(String),
+  `device` LowCardinality(String),
+  `brand` LowCardinality(String),
+  `model` LowCardinality(String),
+  `imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),
+  INDEX idx_name name TYPE bloom_filter GRANULARITY 1,
+  INDEX idx_properties_bounce properties['__bounce'] TYPE set(3) GRANULARITY 1,
+  INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1,
+  INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree()
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (project_id, toDate(created_at), profile_id, name)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE TABLE IF NOT EXISTS events_bots (
+  `id` UUID DEFAULT generateUUIDv4(),
+  `project_id` String,
+  `name` String,
+  `type` String,
+  `path` String,
+  `created_at` DateTime64(3)
+)
+ENGINE = MergeTree()
+ORDER BY (project_id, created_at)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE TABLE IF NOT EXISTS profiles (
+  `id` String CODEC(ZSTD(3)),
+  `is_external` Bool,
+  `first_name` String CODEC(ZSTD(3)),
+  `last_name` String CODEC(ZSTD(3)),
+  `email` String CODEC(ZSTD(3)),
+  `avatar` String CODEC(ZSTD(3)),
+  `properties` Map(String, String) CODEC(ZSTD(3)),
+  `project_id` String CODEC(ZSTD(3)),
+  `created_at` DateTime64(3) CODEC(Delta(4), LZ4),
+  INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
+  INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
+  INDEX idx_email email TYPE bloom_filter GRANULARITY 1
+)
+ENGINE = ReplacingMergeTree(created_at)
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (project_id, id)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE TABLE IF NOT EXISTS profile_aliases (
+  `project_id` String,
+  `profile_id` String,
+  `alias` String,
+  `created_at` DateTime
+)
+ENGINE = MergeTree()
+ORDER BY (project_id, profile_id, alias, created_at)
+SETTINGS index_granularity = 8192;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv
+ENGINE = AggregatingMergeTree()
+PARTITION BY toYYYYMMDD(date)
+ORDER BY (project_id, date)
+AS SELECT
+        toDate(created_at) as date,
+        uniqState(profile_id) as profile_id,
+        project_id
+      FROM events
+      GROUP BY date, project_id;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS cohort_events_mv
+ENGINE = AggregatingMergeTree()
+ORDER BY (project_id, name, created_at, profile_id)
+AS SELECT
+        project_id,
+        name,
+        toDate(created_at) AS created_at,
+        profile_id,
+        COUNT() AS event_count
+      FROM events
+      WHERE profile_id != device_id
+      GROUP BY project_id, name, created_at, profile_id;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS distinct_event_names_mv
+ENGINE = AggregatingMergeTree()
+ORDER BY (project_id, name, created_at)
+AS SELECT
+        project_id,
+        name,
+        max(created_at) AS created_at,
+        count() AS event_count
+      FROM events
+      GROUP BY project_id, name;
+
+---
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS event_property_values_mv
+ENGINE = AggregatingMergeTree()
+ORDER BY (project_id, name, property_key, property_value)
+AS SELECT
+        project_id,
+        name,
+        key_value.keys as property_key,
+        key_value.values as property_value,
+        created_at
+      FROM (
+        SELECT
+          project_id,
+          name,
+          untuple(arrayJoin(properties)) as key_value,
+          max(created_at) as created_at
+        FROM events
+        GROUP BY project_id, name, key_value
+      )
+      WHERE property_value != ''
+        AND property_key != ''
+        AND property_key NOT IN ('__duration_from', '__properties_from')
+      GROUP BY project_id, name, property_key, property_value, created_at;

packages/db/code-migrations/4-add-sessions.ts

@@ -0,0 +1,159 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { formatClickhouseDate } from '../src/clickhouse/client';
+import {
+  createTable,
+  runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+export async function up() {
+  const isClustered = getIsCluster();
+
+  const sqls: string[] = [
+    ...createTable({
+      name: 'sessions',
+      engine: 'VersionedCollapsingMergeTree(sign, version)',
+      columns: [
+        '`id` String',
+        '`project_id` String CODEC(ZSTD(3))',
+        '`profile_id` String CODEC(ZSTD(3))',
+        '`device_id` String CODEC(ZSTD(3))',
+        '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+        '`ended_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+        '`is_bounce` Bool',
+        '`entry_origin` LowCardinality(String)',
+        '`entry_path` String CODEC(ZSTD(3))',
+        '`exit_origin` LowCardinality(String)',
+        '`exit_path` String CODEC(ZSTD(3))',
+        '`screen_view_count` Int32',
+        '`revenue` Float64',
+        '`event_count` Int32',
+        '`duration` UInt32',
+        '`country` LowCardinality(FixedString(2))',
+        '`region` LowCardinality(String)',
+        '`city` String',
+        '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+        '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+        '`device` LowCardinality(String)',
+        '`brand` LowCardinality(String)',
+        '`model` LowCardinality(String)',
+        '`browser` LowCardinality(String)',
+        '`browser_version` LowCardinality(String)',
+        '`os` LowCardinality(String)',
+        '`os_version` LowCardinality(String)',
+        '`utm_medium` String CODEC(ZSTD(3))',
+        '`utm_source` String CODEC(ZSTD(3))',
+        '`utm_campaign` String CODEC(ZSTD(3))',
+        '`utm_content` String CODEC(ZSTD(3))',
+        '`utm_term` String CODEC(ZSTD(3))',
+        '`referrer` String CODEC(ZSTD(3))',
+        '`referrer_name` String CODEC(ZSTD(3))',
+        '`referrer_type` LowCardinality(String)',
+        '`sign` Int8',
+        '`version` UInt64',
+        '`properties` Map(String, String) CODEC(ZSTD(3))',
+      ],
+      orderBy: ['project_id', 'id', 'toDate(created_at)', 'profile_id'],
+      partitionBy: 'toYYYYMM(created_at)',
+      settings: {
+        index_granularity: 8192,
+      },
+      distributionHash:
+        'cityHash64(project_id, toString(toStartOfHour(created_at)))',
+      replicatedVersion: '1',
+      isClustered,
+    }),
+  ];
+
+  sqls.push(...createOldSessions());
+
+  fs.writeFileSync(
+    path.join(__filename.replace('.ts', '.sql')),
+    sqls
+      .map((sql) =>
+        sql
+          .trim()
+          .replace(/;$/, '')
+          .replace(/\n{2,}/g, '\n')
+          .concat(';'),
+      )
+      .join('\n\n---\n\n'),
+  );
+
+  if (!process.argv.includes('--dry')) {
+    await runClickhouseMigrationCommands(sqls);
+  }
+}
+
+function createOldSessions() {
+  let startDate = new Date('2024-03-01');
+  const endDate = new Date();
+  const sqls: string[] = [];
+  while (startDate <= endDate) {
+    const endDate = startDate;
+    startDate = new Date(startDate.getTime() + 1000 * 60 * 60 * 24);
+    sqls.push(`
+      INSERT INTO openpanel.sessions
+        WITH unique_sessions AS (
+          SELECT session_id, min(created_at) as first_event_at
+          FROM openpanel.events
+          WHERE 
+            created_at BETWEEN '${formatClickhouseDate(endDate)}' AND '${formatClickhouseDate(startDate)}'
+            AND session_id != ''
+          GROUP BY session_id
+          HAVING first_event_at >= '${formatClickhouseDate(endDate)}'
+        )
+        SELECT 
+          any(e.session_id) as id,
+          any(e.project_id) as project_id,
+          if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id,
+          any(e.device_id) as device_id,
+          argMin(e.created_at, e.created_at) as created_at,
+          argMax(e.created_at, e.created_at) as ended_at,
+          if(
+            argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = '',
+            if(countIf(e.name = 'screen_view') > 1, true, false),
+            argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = 'true'
+          ) as is_bounce,
+          argMinIf(e.origin, e.created_at, e.name = 'session_start') as entry_origin,
+          argMinIf(e.path, e.created_at, e.name = 'session_start') as entry_path,
+          argMaxIf(e.origin, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_origin,
+          argMaxIf(e.path, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_path,
+          countIf(e.name = 'screen_view') as screen_view_count,
+          0 as revenue,
+          countIf(e.name != 'screen_view' AND e.name != 'session_start' AND e.name != 'session_end') as event_count,
+          sumIf(e.duration, name = 'session_end') AS duration,
+          argMinIf(e.country, e.created_at, e.name = 'session_start') as country,
+          argMinIf(e.region, e.created_at, e.name = 'session_start') as region,
+          argMinIf(e.city, e.created_at, e.name = 'session_start') as city,
+          argMinIf(e.longitude, e.created_at, e.name = 'session_start') as longitude,
+          argMinIf(e.latitude, e.created_at, e.name = 'session_start') as latitude,
+          argMinIf(e.device, e.created_at, e.name = 'session_start') as device,
+          argMinIf(e.brand, e.created_at, e.name = 'session_start') as brand,
+          argMinIf(e.model, e.created_at, e.name = 'session_start') as model,
+          argMinIf(e.browser, e.created_at, e.name = 'session_start') as browser,
+          argMinIf(e.browser_version, e.created_at, e.name = 'session_start') as browser_version,
+          argMinIf(e.os, e.created_at, e.name = 'session_start') as os,
+          argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version,
+          argMinIf(e.properties['__utm_medium'], e.created_at, e.name = 'session_start') as utm_medium,
+          argMinIf(e.properties['__utm_source'], e.created_at, e.name = 'session_start') as utm_source,
+          argMinIf(e.properties['__utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign,
+          argMinIf(e.properties['__utm_content'], e.created_at, e.name = 'session_start') as utm_content,
+          argMinIf(e.properties['__utm_term'], e.created_at, e.name = 'session_start') as utm_term,
+          argMinIf(e.referrer, e.created_at, e.name = 'session_start') as referrer,
+          argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name,
+          argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type,
+          1 as sign,
+          1 as version,
+          argMinIf(e.properties, e.created_at, e.name = 'session_start') as properties
+        FROM events e
+        WHERE 
+          e.session_id IN (SELECT session_id FROM unique_sessions)
+          AND e.created_at BETWEEN '${formatClickhouseDate(endDate)}' AND '${formatClickhouseDate(new Date(startDate.getTime() + 1000 * 60 * 60 * 24 * 3))}'
+        GROUP BY e.session_id
+      `);
+  }
+
+  return sqls;
+}

packages/db/code-migrations/helpers.ts

@@ -3,7 +3,9 @@ export function printBoxMessage(title: string, lines: (string | unknown)[]) {
   console.log('│');
   if (title) {
     console.log(`│  ${title}`);
-    console.log('│');
+    if (lines.length) {
+      console.log('│');
+    }
   }
   lines.forEach((line) => {
     console.log(`│  ${line}`);
@@ -11,3 +13,20 @@ export function printBoxMessage(title: string, lines: (string | unknown)[]) {
   console.log('│');
   console.log('└──┘');
 }
+
+export function getIsCluster() {
+  const args = process.argv;
+  const noClusterArg = args.includes('--no-cluster');
+  if (noClusterArg) {
+    return false;
+  }
+  return !getIsSelfHosting();
+}
+
+export function getIsSelfHosting() {
+  return !!process.env.SELF_HOSTED;
+}
+
+export function getIsDry() {
+  return process.argv.includes('--dry');
+}

packages/db/code-migrations/migrate.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { ch, db } from '../index';
-import { printBoxMessage } from './helpers';
+import { db } from '../index';
+import { getIsDry, getIsSelfHosting, printBoxMessage } from './helpers';
 
 async function migrate() {
   const args = process.argv.slice(2);
@@ -15,11 +15,38 @@ async function migrate() {
     );
   });
 
+  const finishedMigrations = await db.codeMigration.findMany();
+
+  printBoxMessage('📋 Plan', [
+    '\t✅ Finished:',
+    ...finishedMigrations.map(
+      (migration) => `\t- ${migration.name} (${migration.createdAt})`,
+    ),
+    '',
+    '\t🔄 Will run now:',
+    ...migrations
+      .filter(
+        (migration) =>
+          !finishedMigrations.some(
+            (finishedMigration) => finishedMigration.name === migration,
+          ),
+      )
+      .map((migration) => `\t- ${migration}`),
+  ]);
+
+  printBoxMessage('🌍 Environment', [
+    `POSTGRES:   ${process.env.DATABASE_URL}`,
+    `CLICKHOUSE: ${process.env.CLICKHOUSE_URL}`,
+  ]);
+
+  if (!getIsSelfHosting()) {
+    printBoxMessage('🕒 Migrations starts in 10 seconds', []);
+    await new Promise((resolve) => setTimeout(resolve, 10000));
+  }
+
   if (migration) {
     await runMigration(migrationsDir, migration);
   } else {
-    const finishedMigrations = await db.codeMigration.findMany();
-
     for (const file of migrations) {
       if (finishedMigrations.some((migration) => migration.name === file)) {
         printBoxMessage('✅  Already Migrated  ✅', [`${file}`]);
@@ -39,17 +66,19 @@ async function runMigration(migrationsDir: string, file: string) {
   try {
     const migration = await import(path.join(migrationsDir, file));
     await migration.up();
-    await db.codeMigration.upsert({
-      where: {
-        name: file,
-      },
-      update: {
-        name: file,
-      },
-      create: {
-        name: file,
-      },
-    });
+    if (!getIsDry()) {
+      await db.codeMigration.upsert({
+        where: {
+          name: file,
+        },
+        update: {
+          name: file,
+        },
+        create: {
+          name: file,
+        },
+      });
+    }
   } catch (error) {
     printBoxMessage('❌  Migration Failed  ❌', [
       `Error running migration ${file}:`,