improve(self-hosting): remove goose, custom migration, docs, remove zookeeper

2025-02-17 20:54:54 +01:00
parent 16b6786165
commit 329c0885d4
47 changed files with 2181 additions and 1825 deletions
--- a/packages/db/code-migrations/3-init-ch.ts
+++ b/packages/db/code-migrations/3-init-ch.ts
@@ -0,0 +1,372 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { formatClickhouseDate } from '../src/clickhouse/client';
+import {
+  createDatabase,
+  createMaterializedView,
+  createTable,
+  dropTable,
+  getExistingTables,
+  moveDataBetweenTables,
+  renameTable,
+  runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { printBoxMessage } from './helpers';
+
+export async function up() {
+  const replicatedVersion = '1';
+  const existingTables = await getExistingTables();
+  const hasSelfHosting = existingTables.includes('self_hosting_distributed');
+  const hasEvents = existingTables.includes('events_distributed');
+  const hasEventsV2 = existingTables.includes('events_v2');
+  const hasEventsBots = existingTables.includes('events_bots_distributed');
+  const hasProfiles = existingTables.includes('profiles_distributed');
+  const hasProfileAliases = existingTables.includes(
+    'profile_aliases_distributed',
+  );
+
+  const isSelfHosting = !!process.env.SELF_HOSTING;
+  const isClustered = !isSelfHosting;
+
+  const isSelfHostingPostCluster =
+    existingTables.includes('events_replicated') && isSelfHosting;
+
+  const isSelfHostingPreCluster =
+    !isSelfHostingPostCluster &&
+    existingTables.includes('events_v2') &&
+    isSelfHosting;
+
+  const isSelfHostingOld = existingTables.length !== 0 && isSelfHosting;
+
+  const sqls: string[] = [];
+
+  // Move tables to old names if they exists
+  if (isSelfHostingOld) {
+    sqls.push(
+      ...existingTables
+        .filter((table) => {
+          return (
+            !table.endsWith('_tmp') && !existingTables.includes(`${table}_tmp`)
+          );
+        })
+        .flatMap((table) => {
+          return renameTable({
+            from: table,
+            to: `${table}_tmp`,
+            isClustered: false,
+          });
+        }),
+    );
+  }
+
+  sqls.push(
+    createDatabase('openpanel', isClustered),
+    // Create new tables
+    ...createTable({
+      name: 'self_hosting',
+      columns: ['`created_at` Date', '`domain` String', '`count` UInt64'],
+      orderBy: ['domain', 'created_at'],
+      partitionBy: 'toYYYYMM(created_at)',
+      distributionHash: 'cityHash64(domain)',
+      replicatedVersion,
+      isClustered,
+    }),
+    ...createTable({
+      name: 'events',
+      columns: [
+        '`id` UUID DEFAULT generateUUIDv4()',
+        '`name` LowCardinality(String)',
+        '`sdk_name` LowCardinality(String)',
+        '`sdk_version` LowCardinality(String)',
+        '`device_id` String CODEC(ZSTD(3))',
+        '`profile_id` String CODEC(ZSTD(3))',
+        '`project_id` String CODEC(ZSTD(3))',
+        '`session_id` String CODEC(LZ4)',
+        '`path` String CODEC(ZSTD(3))',
+        '`origin` String CODEC(ZSTD(3))',
+        '`referrer` String CODEC(ZSTD(3))',
+        '`referrer_name` String CODEC(ZSTD(3))',
+        '`referrer_type` LowCardinality(String)',
+        '`duration` UInt64 CODEC(Delta(4), LZ4)',
+        '`properties` Map(String, String) CODEC(ZSTD(3))',
+        '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+        '`country` LowCardinality(FixedString(2))',
+        '`city` String',
+        '`region` LowCardinality(String)',
+        '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+        '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)',
+        '`os` LowCardinality(String)',
+        '`os_version` LowCardinality(String)',
+        '`browser` LowCardinality(String)',
+        '`browser_version` LowCardinality(String)',
+        '`device` LowCardinality(String)',
+        '`brand` LowCardinality(String)',
+        '`model` LowCardinality(String)',
+        '`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4)',
+      ],
+      indices: [
+        'INDEX idx_name name TYPE bloom_filter GRANULARITY 1',
+        "INDEX idx_properties_bounce properties['__bounce'] TYPE set(3) GRANULARITY 1",
+        'INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1',
+        'INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1',
+      ],
+      orderBy: ['project_id', 'toDate(created_at)', 'profile_id', 'name'],
+      partitionBy: 'toYYYYMM(created_at)',
+      settings: {
+        index_granularity: 8192,
+      },
+      distributionHash:
+        'cityHash64(project_id, toString(toStartOfHour(created_at)))',
+      replicatedVersion,
+      isClustered,
+    }),
+    ...createTable({
+      name: 'events_bots',
+      columns: [
+        '`id` UUID DEFAULT generateUUIDv4()',
+        '`project_id` String',
+        '`name` String',
+        '`type` String',
+        '`path` String',
+        '`created_at` DateTime64(3)',
+      ],
+      orderBy: ['project_id', 'created_at'],
+      settings: {
+        index_granularity: 8192,
+      },
+      distributionHash:
+        'cityHash64(project_id, toString(toStartOfDay(created_at)))',
+      replicatedVersion,
+      isClustered,
+    }),
+    ...createTable({
+      name: 'profiles',
+      columns: [
+        '`id` String CODEC(ZSTD(3))',
+        '`is_external` Bool',
+        '`first_name` String CODEC(ZSTD(3))',
+        '`last_name` String CODEC(ZSTD(3))',
+        '`email` String CODEC(ZSTD(3))',
+        '`avatar` String CODEC(ZSTD(3))',
+        '`properties` Map(String, String) CODEC(ZSTD(3))',
+        '`project_id` String CODEC(ZSTD(3))',
+        '`created_at` DateTime64(3) CODEC(Delta(4), LZ4)',
+      ],
+      indices: [
+        'INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1',
+        'INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1',
+        'INDEX idx_email email TYPE bloom_filter GRANULARITY 1',
+      ],
+      engine: 'ReplacingMergeTree(created_at)',
+      orderBy: ['project_id', 'id'],
+      partitionBy: 'toYYYYMM(created_at)',
+      settings: {
+        index_granularity: 8192,
+      },
+      distributionHash: 'cityHash64(project_id)',
+      replicatedVersion,
+      isClustered,
+    }),
+    ...createTable({
+      name: 'profile_aliases',
+      columns: [
+        '`project_id` String',
+        '`profile_id` String',
+        '`alias` String',
+        '`created_at` DateTime',
+      ],
+      orderBy: ['project_id', 'profile_id', 'alias', 'created_at'],
+      settings: {
+        index_granularity: 8192,
+      },
+      distributionHash: 'cityHash64(project_id)',
+      replicatedVersion,
+      isClustered,
+    }),
+
+    // Create materialized views
+    ...createMaterializedView({
+      name: 'dau_mv',
+      tableName: 'events',
+      orderBy: ['project_id', 'date'],
+      partitionBy: 'toYYYYMMDD(date)',
+      query: `SELECT
+        toDate(created_at) as date,
+        uniqState(profile_id) as profile_id,
+        project_id
+      FROM {events}
+      GROUP BY date, project_id`,
+      distributionHash: 'cityHash64(project_id, date)',
+      replicatedVersion,
+      isClustered,
+    }),
+    ...createMaterializedView({
+      name: 'cohort_events_mv',
+      tableName: 'events',
+      orderBy: ['project_id', 'name', 'created_at', 'profile_id'],
+      query: `SELECT
+        project_id,
+        name,
+        toDate(created_at) AS created_at,
+        profile_id,
+        COUNT() AS event_count
+      FROM {events}
+      WHERE profile_id != device_id
+      GROUP BY project_id, name, created_at, profile_id`,
+      distributionHash: 'cityHash64(project_id, toString(created_at))',
+      replicatedVersion,
+      isClustered,
+    }),
+    ...createMaterializedView({
+      name: 'distinct_event_names_mv',
+      tableName: 'events',
+      orderBy: ['project_id', 'name', 'created_at'],
+      query: `SELECT
+        project_id,
+        name,
+        max(created_at) AS created_at,
+        count() AS event_count
+      FROM {events}
+      GROUP BY project_id, name`,
+      distributionHash: 'cityHash64(name, created_at)',
+      replicatedVersion,
+      isClustered,
+    }),
+    ...createMaterializedView({
+      name: 'event_property_values_mv',
+      tableName: 'events',
+      orderBy: ['project_id', 'name', 'property_key', 'property_value'],
+      query: `SELECT
+        project_id,
+        name,
+        key_value.keys as property_key,
+        key_value.values as property_value,
+        created_at
+      FROM (
+        SELECT
+          project_id,
+          name,
+          untuple(arrayJoin(properties)) as key_value,
+          max(created_at) as created_at
+        FROM {events}
+        GROUP BY project_id, name, key_value
+      )
+      WHERE property_value != ''
+        AND property_key != ''
+        AND property_key NOT IN ('__duration_from', '__properties_from')
+      GROUP BY project_id, name, property_key, property_value, created_at`,
+      distributionHash: 'cityHash64(project_id, name)',
+      replicatedVersion,
+      isClustered,
+    }),
+  );
+
+  if (isSelfHostingPostCluster) {
+    sqls.push(
+      // Move data between tables
+      ...(hasSelfHosting
+        ? moveDataBetweenTables({
+            from: 'self_hosting_replicated_tmp',
+            to: 'self_hosting',
+            batch: {
+              column: 'created_at',
+              interval: 'month',
+              transform: (date) => {
+                return formatClickhouseDate(date, true);
+              },
+            },
+          })
+        : []),
+      ...(hasProfileAliases
+        ? moveDataBetweenTables({
+            from: 'profile_aliases_replicated_tmp',
+            to: 'profile_aliases',
+            batch: {
+              column: 'created_at',
+              interval: 'month',
+            },
+          })
+        : []),
+      ...(hasEventsBots
+        ? moveDataBetweenTables({
+            from: 'events_bots_replicated_tmp',
+            to: 'events_bots',
+            batch: {
+              column: 'created_at',
+              interval: 'month',
+            },
+          })
+        : []),
+      ...(hasProfiles
+        ? moveDataBetweenTables({
+            from: 'profiles_replicated_tmp',
+            to: 'profiles',
+            batch: {
+              column: 'created_at',
+              interval: 'month',
+            },
+          })
+        : []),
+      ...(hasEvents
+        ? moveDataBetweenTables({
+            from: 'events_replicated_tmp',
+            to: 'events',
+            batch: {
+              column: 'created_at',
+              interval: 'week',
+            },
+          })
+        : []),
+    );
+  }
+
+  if (isSelfHostingPreCluster) {
+    sqls.push(
+      ...(hasEventsV2
+        ? moveDataBetweenTables({
+            from: 'events_v2',
+            to: 'events',
+            batch: {
+              column: 'created_at',
+              interval: 'week',
+            },
+          })
+        : []),
+    );
+  }
+
+  fs.writeFileSync(
+    path.join(__dirname, '3-init-ch.sql'),
+    sqls
+      .map((sql) =>
+        sql
+          .trim()
+          .replace(/;$/, '')
+          .replace(/\n{2,}/g, '\n')
+          .concat(';'),
+      )
+      .join('\n\n---\n\n'),
+  );
+
+  printBoxMessage('Will start migration for self-hosting setup.', [
+    'This will move all data from the old tables to the new ones.',
+    'This might take a while depending on your server.',
+  ]);
+
+  if (!process.argv.includes('--dry')) {
+    await runClickhouseMigrationCommands(sqls);
+  }
+
+  if (isSelfHostingOld) {
+    printBoxMessage(
+      '⚠️ Please run the following command to clean up unused tables:',
+      existingTables.map(
+        (table) =>
+          `docker compose exec -it op-ch clickhouse-client --query "${dropTable(
+            `openpanel.${table}_tmp`,
+            false,
+          )}"`,
+      ),
+    );
+  }
+}
--- a/packages/db/code-migrations/migrate.ts
+++ b/packages/db/code-migrations/migrate.ts
@@ -5,7 +5,7 @@ import { printBoxMessage } from './helpers';

 async function migrate() {
  const args = process.argv.slice(2);
-  const migration = args[0];
+  const migration = args.filter((arg) => !arg.startsWith('--'))[0];

  const migrationsDir = path.join(__dirname, '..', 'code-migrations');
  const migrations = fs.readdirSync(migrationsDir).filter((file) => {
@@ -22,7 +22,7 @@ async function migrate() {

    for (const file of migrations) {
      if (finishedMigrations.some((migration) => migration.name === file)) {
-        printBoxMessage('⏭️  Skipping Migration  ⏭️', [`${file}`]);
+        printBoxMessage('✅  Already Migrated  ✅', [`${file}`]);
        continue;
      }

--- a/packages/db/index.ts
+++ b/packages/db/index.ts
@@ -1,5 +1,5 @@
 export * from './src/prisma-client';
-export * from './src/clickhouse-client';
+export * from './src/clickhouse/client';
 export * from './src/sql-builder';
 export * from './src/services/chart.service';
 export * from './src/services/clients.service';
--- a/packages/db/migrations/20240906185616_init.sql
+++ b/packages/db/migrations/20240906185616_init.sql
@@ -1,112 +0,0 @@
-- +goose Up
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS self_hosting
-(
-    created_at Date,
-    domain String,
-    count UInt64
-)
-ENGINE = MergeTree()
-ORDER BY (domain, created_at)
-PARTITION BY toYYYYMM(created_at);
-- +goose StatementEnd
-
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS events_v2 (
-  `id` UUID DEFAULT generateUUIDv4(),
-  `name` String,
-  `sdk_name` String,
-  `sdk_version` String,
-  `device_id` String,
-  `profile_id` String,
-  `project_id` String,
-  `session_id` String,
-  `path` String,
-  `origin` String,
-  `referrer` String,
-  `referrer_name` String,
-  `referrer_type` String,
-  `duration` UInt64,
-  `properties` Map(String, String),
-  `created_at` DateTime64(3),
-  `country` String,
-  `city` String,
-  `region` String,
-  `longitude` Nullable(Float32),
-  `latitude` Nullable(Float32),
-  `os` String,
-  `os_version` String,
-  `browser` String,
-  `browser_version` String,
-  `device` String,
-  `brand` String,
-  `model` String,
-  `imported_at` Nullable(DateTime),
-  INDEX idx_name name TYPE bloom_filter GRANULARITY 1,
-  INDEX idx_properties_bounce properties ['__bounce'] TYPE set (3) GRANULARITY 1,
-  INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1,
-  INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1
-) ENGINE = MergeTree PARTITION BY toYYYYMM(created_at)
-ORDER BY
-  (project_id, toDate(created_at), profile_id, name) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS events_bots (
-  `id` UUID DEFAULT generateUUIDv4(),
-  `project_id` String,
-  `name` String,
-  `type` String,
-  `path` String,
-  `created_at` DateTime64(3)
-) ENGINE MergeTree
-ORDER BY
-  (project_id, created_at) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS profiles (
-  `id` String,
-  `is_external` Bool,
-  `first_name` String,
-  `last_name` String,
-  `email` String,
-  `avatar` String,
-  `properties` Map(String, String),
-  `project_id` String,
-  `created_at` DateTime
-) ENGINE = ReplacingMergeTree(created_at)
-ORDER BY
-  (id) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS profile_aliases (
-  `project_id` String,
-  `profile_id` String,
-  `alias` String,
-  `created_at` DateTime
-) ENGINE = MergeTree
-ORDER BY
-  (project_id, profile_id, alias, created_at) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMMDD(date)
-ORDER BY
-  (project_id, date) POPULATE AS
-SELECT
-  toDate(created_at) as date,
-  uniqState(profile_id) as profile_id,
-  project_id
-FROM
-  events_v2
-GROUP BY
-  date,
-  project_id;
-- +goose StatementEnd
-
-- +goose Down
-- +goose StatementBegin
-SELECT 'down SQL query';
-- +goose StatementEnd
--- a/packages/db/migrations/20240907202846_optimize_profiles.sql
+++ b/packages/db/migrations/20240907202846_optimize_profiles.sql
@@ -1,44 +0,0 @@
-- +goose Up
-- +goose StatementBegin
-CREATE TABLE profiles_tmp
-(
-    `id` String,
-    `is_external` Bool,
-    `first_name` String,
-    `last_name` String,
-    `email` String,
-    `avatar` String,
-    `properties` Map(String, String),
-    `project_id` String,
-    `created_at` DateTime,
-    INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
-    INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
-    INDEX idx_email email TYPE bloom_filter GRANULARITY 1
-)
-ENGINE = ReplacingMergeTree(created_at)
-PARTITION BY toYYYYMM(created_at)
-ORDER BY (project_id, created_at, id)
-SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO profiles_tmp SELECT
-    id,
-    is_external,
-    first_name,
-    last_name,
-    email,
-    avatar,
-    properties,
-    project_id,
-    created_at
-FROM profiles;
-- +goose StatementEnd
-- +goose StatementBegin
-OPTIMIZE TABLE profiles_tmp FINAL;
-- +goose StatementEnd
-- +goose StatementBegin
-RENAME TABLE profiles TO profiles_old, profiles_tmp TO profiles;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE profiles_old;
-- +goose StatementEnd
--- a/packages/db/migrations/20240917184138_fix_profile_merges.sql
+++ b/packages/db/migrations/20240917184138_fix_profile_merges.sql
@@ -1,55 +0,0 @@
-- +goose Up
-- +goose StatementBegin
-CREATE TABLE profiles_fixed
-(
-    `id` String,
-    `is_external` Bool,
-    `first_name` String,
-    `last_name` String,
-    `email` String,
-    `avatar` String,
-    `properties` Map(String, String),
-    `project_id` String,
-    `created_at` DateTime,
-    INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
-    INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
-    INDEX idx_email email TYPE bloom_filter GRANULARITY 1
-)
-ENGINE = ReplacingMergeTree(created_at)
-PARTITION BY toYYYYMM(created_at)
-ORDER BY (project_id, id)
-SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-
-- +goose StatementBegin
-INSERT INTO profiles_fixed SELECT
-    id,
-    is_external,
-    first_name,
-    last_name,
-    email,
-    avatar,
-    properties,
-    project_id,
-    created_at
-FROM profiles;
-- +goose StatementEnd
-
-- +goose StatementBegin
-OPTIMIZE TABLE profiles_fixed FINAL;
-- +goose StatementEnd
-
-- +goose StatementBegin
-RENAME TABLE profiles TO profiles_old, profiles_fixed TO profiles;
-- +goose StatementEnd
-
-- +goose StatementBegin
-DROP TABLE profiles_old;
-- +goose StatementEnd
-
-- +goose Down
-- +goose StatementBegin
-- This is a destructive migration, so the down migration is not provided.
-- If needed, you should restore from a backup.
-SELECT 'down migration not implemented';
-- +goose StatementEnd
--- a/packages/db/migrations/20241007210706_retention_mv.sql
+++ b/packages/db/migrations/20241007210706_retention_mv.sql
@@ -1,58 +0,0 @@
-- +goose Up
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW cohort_events_mv ENGINE = AggregatingMergeTree()
-ORDER BY (project_id, name, created_at, profile_id) POPULATE AS
-SELECT project_id,
-    name,
-    toDate(created_at) AS created_at,
-    profile_id,
-    COUNT() AS event_count
-FROM events_v2
-WHERE profile_id != device_id
-GROUP BY project_id,
-    name,
-    created_at,
-    profile_id;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW distinct_event_names_mv ENGINE = AggregatingMergeTree()
-ORDER BY (project_id, name, created_at) POPULATE AS
-SELECT project_id,
-    name,
-    max(created_at) AS created_at,
-    count() AS event_count
-FROM events_v2
-GROUP BY project_id,
-    name;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW event_property_values_mv ENGINE = AggregatingMergeTree()
-ORDER BY (project_id, name, property_key, property_value) POPULATE AS
-select project_id,
-    name,
-    key_value.keys as property_key,
-    key_value.values as property_value,
-    created_at
-from (
-        SELECT project_id,
-            name,
-            untuple(arrayJoin(properties)) as key_value,
-            max(created_at) as created_at
-        from events_v2
-        group by project_id,
-            name,
-            key_value
-    )
-where property_value != ''
-    and property_key != ''
-    and property_key NOT IN ('__duration_from', '__properties_from')
-group by project_id,
-    name,
-    property_key,
-    property_value,
-    created_at;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-SELECT 'down SQL query';
-- +goose StatementEnd
--- a/packages/db/migrations/20241127133914_cluster_prep.sql
+++ b/packages/db/migrations/20241127133914_cluster_prep.sql
@@ -1,351 +0,0 @@
-- +goose Up
-- +goose StatementBegin
-CREATE DATABASE IF NOT EXISTS openpanel;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS self_hosting_replicated ON CLUSTER '{cluster}' (
-  created_at Date,
-  domain String,
-  count UInt64
-) ENGINE = ReplicatedMergeTree(
-  '/clickhouse/tables/{shard}/self_hosting_replicated',
-  '{replica}'
-)
-ORDER BY (domain, created_at) PARTITION BY toYYYYMM(created_at);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS events_replicated ON CLUSTER '{cluster}' (
-  `id` UUID DEFAULT generateUUIDv4(),
-  `name` String,
-  `sdk_name` String,
-  `sdk_version` String,
-  `device_id` String,
-  `profile_id` String,
-  `project_id` String,
-  `session_id` String,
-  `path` String,
-  `origin` String,
-  `referrer` String,
-  `referrer_name` String,
-  `referrer_type` String,
-  `duration` UInt64,
-  `properties` Map(String, String),
-  `created_at` DateTime64(3),
-  `country` String,
-  `city` String,
-  `region` String,
-  `longitude` Nullable(Float32),
-  `latitude` Nullable(Float32),
-  `os` String,
-  `os_version` String,
-  `browser` String,
-  `browser_version` String,
-  `device` String,
-  `brand` String,
-  `model` String,
-  `imported_at` Nullable(DateTime),
-  INDEX idx_name name TYPE bloom_filter GRANULARITY 1,
-  INDEX idx_properties_bounce properties ['__bounce'] TYPE
-  set(3) GRANULARITY 1,
-    INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1,
-    INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1
-) ENGINE = ReplicatedMergeTree(
-  '/clickhouse/tables/{shard}/events_replicated',
-  '{replica}'
-) PARTITION BY toYYYYMM(created_at)
-ORDER BY (project_id, toDate(created_at), profile_id, name) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS events_bots_replicated ON CLUSTER '{cluster}' (
-  `id` UUID DEFAULT generateUUIDv4(),
-  `project_id` String,
-  `name` String,
-  `type` String,
-  `path` String,
-  `created_at` DateTime64(3)
-) ENGINE = ReplicatedMergeTree(
-  '/clickhouse/tables/{shard}/events_bots_replicated',
-  '{replica}'
-)
-ORDER BY (project_id, created_at) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS profiles_replicated ON CLUSTER '{cluster}' (
-  `id` String,
-  `is_external` Bool,
-  `first_name` String,
-  `last_name` String,
-  `email` String,
-  `avatar` String,
-  `properties` Map(String, String),
-  `project_id` String,
-  `created_at` DateTime,
-  INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
-  INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
-  INDEX idx_email email TYPE bloom_filter GRANULARITY 1
-) ENGINE = ReplicatedReplacingMergeTree(
-  '/clickhouse/tables/{shard}/profiles_replicated',
-  '{replica}',
-  created_at
-) PARTITION BY toYYYYMM(created_at)
-ORDER BY (project_id, id) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS profile_aliases_replicated ON CLUSTER '{cluster}' (
-  `project_id` String,
-  `profile_id` String,
-  `alias` String,
-  `created_at` DateTime
-) ENGINE = ReplicatedMergeTree(
-  '/clickhouse/tables/{shard}/profile_aliases_replicated',
-  '{replica}'
-)
-ORDER BY (project_id, profile_id, alias, created_at) SETTINGS index_granularity = 8192;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv_replicated ON CLUSTER '{cluster}' ENGINE = ReplicatedAggregatingMergeTree(
-  '/clickhouse/tables/{shard}/dau_mv_replicated',
-  '{replica}'
-) PARTITION BY toYYYYMMDD(date)
-ORDER BY (project_id, date) AS
-SELECT toDate(created_at) as date,
-  uniqState(profile_id) as profile_id,
-  project_id
-FROM events_replicated
-GROUP BY date,
-  project_id;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW IF NOT EXISTS cohort_events_mv_replicated ON CLUSTER '{cluster}' ENGINE = ReplicatedAggregatingMergeTree(
-  '/clickhouse/tables/{shard}/cohort_events_mv_replicated',
-  '{replica}'
-)
-ORDER BY (project_id, name, created_at, profile_id) AS
-SELECT project_id,
-  name,
-  toDate(created_at) AS created_at,
-  profile_id,
-  COUNT() AS event_count
-FROM events_replicated
-WHERE profile_id != device_id
-GROUP BY project_id,
-  name,
-  created_at,
-  profile_id;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW IF NOT EXISTS distinct_event_names_mv_replicated ON CLUSTER '{cluster}' ENGINE = ReplicatedAggregatingMergeTree(
-  '/clickhouse/tables/{shard}/distinct_event_names_mv_replicated',
-  '{replica}'
-)
-ORDER BY (project_id, name, created_at) AS
-SELECT project_id,
-  name,
-  max(created_at) AS created_at,
-  count() AS event_count
-FROM events_replicated
-GROUP BY project_id,
-  name;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE MATERIALIZED VIEW IF NOT EXISTS event_property_values_mv_replicated ON CLUSTER '{cluster}' ENGINE = ReplicatedAggregatingMergeTree(
-  '/clickhouse/tables/{shard}/event_property_values_mv_replicated',
-  '{replica}'
-)
-ORDER BY (project_id, name, property_key, property_value) AS
-SELECT project_id,
-  name,
-  key_value.keys as property_key,
-  key_value.values as property_value,
-  created_at
-FROM (
-    SELECT project_id,
-      name,
-      untuple(arrayJoin(properties)) as key_value,
-      max(created_at) as created_at
-    FROM events_replicated
-    GROUP BY project_id,
-      name,
-      key_value
-  )
-WHERE property_value != ''
-  AND property_key != ''
-  AND property_key NOT IN ('__duration_from', '__properties_from')
-GROUP BY project_id,
-  name,
-  property_key,
-  property_value,
-  created_at;
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS self_hosting_distributed ON CLUSTER '{cluster}' AS self_hosting_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  self_hosting_replicated,
-  cityHash64(domain)
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS events_distributed ON CLUSTER '{cluster}' AS events_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  events_replicated,
-  cityHash64(project_id, toString(toStartOfHour(created_at)))
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS events_bots_distributed ON CLUSTER '{cluster}' AS events_bots_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  events_bots_replicated,
-  cityHash64(project_id, toString(toStartOfDay(created_at)))
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS profiles_distributed ON CLUSTER '{cluster}' AS profiles_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  profiles_replicated,
-  cityHash64(project_id)
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS dau_mv_distributed ON CLUSTER '{cluster}' AS dau_mv_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  dau_mv_replicated,
-  rand()
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS cohort_events_mv_distributed ON CLUSTER '{cluster}' AS cohort_events_mv_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  cohort_events_mv_replicated,
-  rand()
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS distinct_event_names_mv_distributed ON CLUSTER '{cluster}' AS distinct_event_names_mv_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  distinct_event_names_mv_replicated,
-  rand()
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS event_property_values_mv_distributed ON CLUSTER '{cluster}' AS event_property_values_mv_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  event_property_values_mv_replicated,
-  rand()
-);
-- +goose StatementEnd
-- +goose StatementBegin
-CREATE TABLE IF NOT EXISTS profile_aliases_distributed ON CLUSTER '{cluster}' AS profile_aliases_replicated ENGINE = Distributed(
-  '{cluster}',
-  openpanel,
-  profile_aliases_replicated,
-  cityHash64(project_id)
-);
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO events_replicated
-SELECT *
-FROM events_v2;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO events_bots_replicated
-SELECT *
-FROM events_bots;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO profiles_replicated
-SELECT *
-FROM profiles;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO profile_aliases_replicated
-SELECT *
-FROM profile_aliases;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO self_hosting_replicated
-SELECT *
-FROM self_hosting;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO dau_mv_replicated
-SELECT *
-FROM dau_mv;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO cohort_events_mv_replicated
-SELECT *
-FROM cohort_events_mv;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO distinct_event_names_mv_replicated
-SELECT *
-FROM distinct_event_names_mv;
-- +goose StatementEnd
-- +goose StatementBegin
-INSERT INTO event_property_values_mv_replicated
-SELECT *
-FROM event_property_values_mv;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-DROP TABLE IF EXISTS events_distributed ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS events_bots_distributed ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS profiles_distributed ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS events_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS events_bots_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS profiles_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS profile_aliases_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS dau_mv_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS cohort_events_mv_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS distinct_event_names_mv_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS event_property_values_mv_replicated ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS dau_mv_distributed ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS cohort_events_mv_distributed ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS distinct_event_names_mv_distributed ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-- +goose StatementBegin
-DROP TABLE IF EXISTS event_property_values_mv_distributed ON CLUSTER '{cluster}' SYNC;
-- +goose StatementEnd
-TRUNCATE TABLE events_replicated;
-TRUNCATE TABLE events_bots_replicated;
-TRUNCATE TABLE profiles_replicated;
-TRUNCATE TABLE profile_aliases_replicated;
-TRUNCATE TABLE self_hosting_replicated;
-TRUNCATE TABLE dau_mv_replicated;
-TRUNCATE TABLE cohort_events_mv_replicated;
-TRUNCATE TABLE distinct_event_names_mv_replicated;
-TRUNCATE TABLE event_property_values_mv_replicated;
--- a/packages/db/migrations/goose
+++ b/packages/db/migrations/goose
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-
-if [ -n "$CLICKHOUSE_URL_DIRECT" ]; then
-  export GOOSE_DBSTRING=$CLICKHOUSE_URL_DIRECT
-elif [ -z "$CLICKHOUSE_URL" ]; then
-  echo "Neither CLICKHOUSE_URL_DIRECT nor CLICKHOUSE_URL is set"
-  exit 1
-else
-  export GOOSE_DBSTRING=$CLICKHOUSE_URL
-fi
-
-echo "Clickhouse migration script"
-echo ""
-echo "================="
-echo "Selected database: $GOOSE_DBSTRING"
-echo "================="
-echo ""
-if [ "$1" != "create" ] && [ -z "$CI" ]; then
-  read -p "Are you sure you want to run migrations on this database? (y/n) " -n 1 -r
-  echo
-  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-    echo "Migration cancelled."
-    exit 0
-  fi
-fi
-
-goose clickhouse --dir ./migrations $@
--- a/packages/db/package.json
+++ b/packages/db/package.json
@@ -3,13 +3,11 @@
  "version": "0.0.1",
  "main": "index.ts",
  "scripts": {
-    "goose": "pnpm with-env ./migrations/goose",
    "codegen": "pnpm with-env prisma generate",
    "migrate": "pnpm with-env prisma migrate dev",
-    "migrate:deploy:db:code": "pnpm with-env jiti ./code-migrations/migrate.ts",
+    "migrate:deploy:code": "pnpm with-env jiti ./code-migrations/migrate.ts",
    "migrate:deploy:db": "pnpm with-env prisma migrate deploy",
-    "migrate:deploy:ch": "pnpm goose up",
-    "migrate:deploy": "pnpm migrate:deploy:db && pnpm migrate:deploy:db:code && pnpm migrate:deploy:ch",
+    "migrate:deploy": "pnpm migrate:deploy:db && pnpm migrate:deploy:code",
    "typecheck": "tsc --noEmit",
    "with-env": "dotenv -e ../../.env -c --"
  },
--- a/packages/db/src/buffers/bot-buffer-redis.ts
+++ b/packages/db/src/buffers/bot-buffer-redis.ts
@@ -1,7 +1,7 @@
 import { type Redis, getRedisCache, runEvery } from '@openpanel/redis';

 import { getSafeJson } from '@openpanel/common';
-import { TABLE_NAMES, ch } from '../clickhouse-client';
+import { TABLE_NAMES, ch } from '../clickhouse/client';
 import type { IClickhouseBotEvent } from '../services/event.service';
 import { BaseBuffer } from './base-buffer';

--- a/packages/db/src/buffers/buffer.ts
+++ b/packages/db/src/buffers/buffer.ts
@@ -1,266 +0,0 @@
-import { generateId, getSafeJson } from '@openpanel/common';
-import type { ILogger } from '@openpanel/logger';
-import { createLogger } from '@openpanel/logger';
-import { getRedisCache } from '@openpanel/redis';
-import { pathOr } from 'ramda';
-
-export type Find<T, R = unknown> = (
-  callback: (item: T) => boolean,
-) => Promise<R | null>;
-
-export type FindMany<T, R = unknown> = (
-  callback: (item: T) => boolean,
-) => Promise<R[]>;
-
-export class RedisBuffer<T> {
-  public name: string;
-  protected prefix = 'op:buffer';
-  protected bufferKey: string;
-  private lockKey: string;
-  protected maxBufferSize: number | null;
-  protected logger: ILogger;
-
-  constructor(bufferName: string, maxBufferSize: number | null) {
-    this.name = bufferName;
-    this.bufferKey = bufferName;
-    this.lockKey = `lock:${bufferName}`;
-    this.maxBufferSize = maxBufferSize;
-    this.logger = createLogger({ name: 'buffer' }).child({
-      buffer: bufferName,
-    });
-  }
-
-  protected getKey(name?: string) {
-    const key = `${this.prefix}:${this.bufferKey}`;
-    if (name) {
-      return `${key}:${name}`;
-    }
-    return key;
-  }
-
-  async add(item: T): Promise<void> {
-    try {
-      this.onAdd(item);
-      await getRedisCache().rpush(this.getKey(), JSON.stringify(item));
-      const bufferSize = await getRedisCache().llen(this.getKey());
-
-      this.logger.debug(
-        `Item added (${pathOr('unknown', ['id'], item)}) Current size: ${bufferSize}`,
-      );
-
-      if (this.maxBufferSize && bufferSize >= this.maxBufferSize) {
-        await this.tryFlush();
-      }
-    } catch (error) {
-      this.logger.error('Failed to add item to buffer', { error, item });
-    }
-  }
-
-  public async tryFlush(): Promise<void> {
-    const lockId = generateId();
-    const acquired = await getRedisCache().set(
-      this.lockKey,
-      lockId,
-      'EX',
-      60,
-      'NX',
-    );
-
-    if (acquired === 'OK') {
-      this.logger.info(`Lock acquired. Attempting to flush. ID: ${lockId}`);
-      try {
-        await this.flush();
-      } catch (error) {
-        this.logger.error(`Failed to flush buffer. ID: ${lockId}`, { error });
-      } finally {
-        this.logger.info(`Releasing lock. ID: ${lockId}`);
-        await this.releaseLock(lockId);
-      }
-    } else {
-      this.logger.warn(`Failed to acquire lock. Skipping flush. ID: ${lockId}`);
-    }
-  }
-
-  protected async waitForReleasedLock(
-    maxWaitTime = 8000,
-    checkInterval = 250,
-  ): Promise<boolean> {
-    const startTime = performance.now();
-
-    while (performance.now() - startTime < maxWaitTime) {
-      const lock = await getRedisCache().get(this.lockKey);
-      if (!lock) {
-        return true;
-      }
-
-      await new Promise((resolve) => setTimeout(resolve, checkInterval));
-    }
-
-    this.logger.warn('Timeout waiting for lock release');
-    return false;
-  }
-
-  private async retryOnce(cb: () => Promise<void>) {
-    try {
-      await cb();
-    } catch (e) {
-      this.logger.error(`#1 Failed to execute callback: ${cb.name}`, e);
-      await new Promise((resolve) => setTimeout(resolve, 1000));
-      try {
-        await cb();
-      } catch (e) {
-        this.logger.error(`#2 Failed to execute callback: ${cb.name}`, e);
-        throw e;
-      }
-    }
-  }
-
-  private async flush(): Promise<void> {
-    // Use a transaction to ensure atomicity
-    const result = await getRedisCache()
-      .multi()
-      .lrange(this.getKey(), 0, -1)
-      .lrange(this.getKey('backup'), 0, -1)
-      .del(this.getKey())
-      .exec();
-
-    if (!result) {
-      this.logger.error('No result from redis transaction', {
-        result,
-      });
-      throw new Error('Redis transaction failed');
-    }
-
-    const lrange = result[0];
-    const lrangePrevious = result[1];
-
-    if (!lrange || lrange[0] instanceof Error) {
-      this.logger.error('Error from lrange', {
-        result,
-      });
-      throw new Error('Redis transaction failed');
-    }
-
-    const items = lrange[1] as string[];
-    if (
-      lrangePrevious &&
-      lrangePrevious[0] === null &&
-      Array.isArray(lrangePrevious[1])
-    ) {
-      items.push(...(lrangePrevious[1] as string[]));
-    }
-
-    const parsedItems = items
-      .map((item) => getSafeJson<T | null>(item) as T | null)
-      .filter((item): item is T => item !== null);
-
-    if (parsedItems.length === 0) {
-      this.logger.debug('No items to flush');
-      // Clear any existing backup since we have no items to process
-      await getRedisCache().del(this.getKey('backup'));
-      return;
-    }
-
-    this.logger.info(`Flushing ${parsedItems.length} items`);
-
-    try {
-      // Create backup before processing
-      await getRedisCache().del(this.getKey('backup')); // Clear any existing backup first
-      await getRedisCache().lpush(
-        this.getKey('backup'),
-        ...parsedItems.map((item) => JSON.stringify(item)),
-      );
-
-      const { toInsert, toKeep } = await this.processItems(parsedItems);
-
-      if (toInsert.length) {
-        await this.retryOnce(() => this.insertIntoDB(toInsert));
-        this.onInsert(toInsert);
-      }
-
-      // Add back items to keep
-      if (toKeep.length > 0) {
-        await getRedisCache().lpush(
-          this.getKey(),
-          ...toKeep.map((item) => JSON.stringify(item)),
-        );
-      }
-
-      // Clear backup
-      await getRedisCache().del(this.getKey('backup'));
-
-      this.logger.info(
-        `Inserted ${toInsert.length} items into DB, kept ${toKeep.length} items in buffer`,
-        {
-          toInsert: toInsert.length,
-          toKeep: toKeep.length,
-        },
-      );
-    } catch (error) {
-      this.logger.error('Failed to process queue while flushing buffer', {
-        error,
-        queueSize: parsedItems.length,
-      });
-
-      if (parsedItems.length > 0) {
-        // Add back items to keep
-        this.logger.info('Adding all items back to buffer');
-        await getRedisCache().lpush(
-          this.getKey(),
-          ...parsedItems.map((item) => JSON.stringify(item)),
-        );
-      }
-
-      // Clear the backup since we're adding items back to main buffer
-      await getRedisCache().del(this.getKey('backup'));
-    }
-  }
-
-  private async releaseLock(lockId: string): Promise<void> {
-    this.logger.debug(`Released lock for ${this.getKey()}`);
-    const script = `
-      if redis.call("get", KEYS[1]) == ARGV[1] then
-        return redis.call("del", KEYS[1])
-      else
-        return 0
-      end
-    `;
-    await getRedisCache().eval(script, 1, this.lockKey, lockId);
-  }
-
-  protected async getQueue(count?: number): Promise<T[]> {
-    try {
-      const items = await getRedisCache().lrange(this.getKey(), 0, count ?? -1);
-      return items
-        .map((item) => getSafeJson<T | null>(item) as T | null)
-        .filter((item): item is T => item !== null);
-    } catch (error) {
-      this.logger.error('Failed to get queue', { error });
-      return [];
-    }
-  }
-
-  protected processItems(items: T[]): Promise<{ toInsert: T[]; toKeep: T[] }> {
-    return Promise.resolve({ toInsert: items, toKeep: [] });
-  }
-
-  protected insertIntoDB(_items: T[]): Promise<void> {
-    throw new Error('Not implemented');
-  }
-
-  protected onAdd(_item: T): void {
-    // Override in subclass
-  }
-
-  protected onInsert(_item: T[]): void {
-    // Override in subclass
-  }
-
-  public findMany: FindMany<T, unknown> = () => {
-    return Promise.resolve([]);
-  };
-
-  public find: Find<T, unknown> = () => {
-    return Promise.resolve(null);
-  };
-}
--- a/packages/db/src/buffers/event-buffer-redis.ts
+++ b/packages/db/src/buffers/event-buffer-redis.ts
@@ -5,7 +5,7 @@ import {
  getRedisPub,
  runEvery,
 } from '@openpanel/redis';
-import { ch } from '../clickhouse-client';
+import { ch } from '../clickhouse/client';
 import {
  type IClickhouseEvent,
  type IServiceEvent,
--- a/packages/db/src/buffers/profile-buffer-redis.ts
+++ b/packages/db/src/buffers/profile-buffer-redis.ts
@@ -3,7 +3,7 @@ import { getSafeJson } from '@openpanel/common';
 import { type Redis, getRedisCache } from '@openpanel/redis';
 import { dissocPath, mergeDeepRight, omit, whereEq } from 'ramda';

-import { TABLE_NAMES, ch, chQuery } from '../clickhouse-client';
+import { TABLE_NAMES, ch, chQuery } from '../clickhouse/client';
 import type { IClickhouseProfile } from '../services/profile.service';
 import { BaseBuffer } from './base-buffer';
 import { isPartialMatch } from './partial-json-match';
--- a/packages/db/src/clickhouse/client.ts
+++ b/packages/db/src/clickhouse/client.ts
--- a/packages/db/src/clickhouse/migration.ts
+++ b/packages/db/src/clickhouse/migration.ts
@@ -0,0 +1,454 @@
+import crypto from 'node:crypto';
+import { createClient } from './client';
+import { formatClickhouseDate } from './client';
+
+interface CreateTableOptions {
+  name: string;
+  columns: string[];
+  indices?: string[];
+  engine?: string;
+  orderBy: string[];
+  partitionBy?: string;
+  settings?: Record<string, string | number>;
+  distributionHash: string;
+  replicatedVersion: string;
+  isClustered: boolean;
+}
+
+interface CreateMaterializedViewOptions {
+  name: string;
+  tableName: string;
+  query: string;
+  engine?: string;
+  orderBy: string[];
+  partitionBy?: string;
+  settings?: Record<string, string | number>;
+  populate?: boolean;
+  distributionHash: string;
+  replicatedVersion: string;
+  isClustered: boolean;
+}
+
+const CLUSTER_REPLICA_PATH =
+  '/clickhouse/{installation}/{cluster}/tables/{shard}/openpanel/v{replicatedVersion}/{table}';
+
+const replicated = (tableName: string) => `${tableName}_replicated`;
+
+export const chMigrationClient = createClient({
+  url: process.env.CLICKHOUSE_URL,
+  request_timeout: 3600000, // 1 hour in milliseconds
+  keep_alive: {
+    enabled: true,
+    idle_socket_ttl: 8000,
+  },
+  compression: {
+    request: true,
+  },
+  clickhouse_settings: {
+    wait_end_of_query: 1,
+    // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection.
+    send_progress_in_http_headers: 1,
+    // The interval of sending these progress headers. Here it is less than 60s,
+    http_headers_progress_interval_ms: '50000',
+  },
+});
+
+export function createDatabase(name: string, isClustered: boolean) {
+  if (isClustered) {
+    return `CREATE DATABASE IF NOT EXISTS ${name} ON CLUSTER '{cluster}'`;
+  }
+
+  return `CREATE DATABASE IF NOT EXISTS ${name}`;
+}
+
+/**
+ * Creates SQL statements for table creation in ClickHouse
+ * Handles both clustered and non-clustered scenarios
+ */
+export function createTable({
+  name: tableName,
+  columns,
+  indices = [],
+  engine = 'MergeTree()',
+  orderBy = ['tuple()'],
+  partitionBy,
+  settings = {},
+  distributionHash,
+  replicatedVersion,
+  isClustered,
+}: CreateTableOptions): string[] {
+  const columnDefinitions = [...columns, ...indices].join(',\n  ');
+
+  const settingsClause = Object.entries(settings).length
+    ? `SETTINGS ${Object.entries(settings)
+        .map(([key, value]) => `${key} = ${value}`)
+        .join(', ')}`
+    : '';
+
+  const partitionByClause = partitionBy ? `PARTITION BY ${partitionBy}` : '';
+
+  if (!isClustered) {
+    // Non-clustered scenario: single table
+    return [
+      `CREATE TABLE IF NOT EXISTS ${tableName} (
+  ${columnDefinitions}
+)
+ENGINE = ${engine}
+${partitionByClause}
+ORDER BY (${orderBy.join(', ')})
+${settingsClause}`.trim(),
+    ];
+  }
+
+  return [
+    // Local replicated table
+    `CREATE TABLE IF NOT EXISTS ${replicated(tableName)} ON CLUSTER '{cluster}' (
+  ${columnDefinitions}
+)
+ENGINE = Replicated${engine.replace(/^(.+?)\((.+?)?\)/, `$1('${CLUSTER_REPLICA_PATH.replace('{replicatedVersion}', replicatedVersion)}', '{replica}', $2)`).replace(/, \)$/, ')')}
+${partitionByClause}
+ORDER BY (${orderBy.join(', ')})
+${settingsClause}`.trim(),
+    // Distributed table
+    `CREATE TABLE IF NOT EXISTS ${tableName} ON CLUSTER '{cluster}' AS ${replicated(tableName)}
+ENGINE = Distributed('{cluster}', currentDatabase(), ${replicated(tableName)}, ${distributionHash})`,
+  ];
+}
+
+/**
+ * Generates ALTER TABLE statements for adding columns
+ */
+export function addColumns(
+  tableName: string,
+  columns: string[],
+  isClustered: boolean,
+): string[] {
+  if (isClustered) {
+    return columns.map(
+      (col) =>
+        `ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' ADD COLUMN IF NOT EXISTS ${col}`,
+    );
+  }
+
+  return columns.map(
+    (col) => `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS ${col}`,
+  );
+}
+
+/**
+ * Generates ALTER TABLE statements for dropping columns
+ */
+export function dropColumns(
+  tableName: string,
+  columnNames: string[],
+  isClustered: boolean,
+): string[] {
+  if (isClustered) {
+    return columnNames.map(
+      (colName) =>
+        `ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' DROP COLUMN IF EXISTS ${colName}`,
+    );
+  }
+
+  return columnNames.map(
+    (colName) => `ALTER TABLE ${tableName} DROP COLUMN IF EXISTS ${colName}`,
+  );
+}
+
+export async function getExistingTables() {
+  try {
+    const existingTablesQuery = await chMigrationClient.query({
+      query: `SELECT name FROM system.tables WHERE database = 'openpanel'`,
+      format: 'JSONEachRow',
+    });
+    return (await existingTablesQuery.json<{ name: string }>())
+      .map((table) => table.name)
+      .filter((table) => !table.includes('.inner_id'));
+  } catch (e) {
+    console.error(e);
+    return [];
+  }
+}
+
+export function renameTable({
+  from,
+  to,
+  isClustered,
+}: {
+  from: string;
+  to: string;
+  isClustered: boolean;
+}) {
+  if (isClustered) {
+    return [
+      `RENAME TABLE ${replicated(from)} TO ${replicated(to)} ON CLUSTER '{cluster}'`,
+      `RENAME TABLE ${from} TO ${to} ON CLUSTER '{cluster}'`,
+    ];
+  }
+
+  return [`RENAME TABLE ${from} TO ${to}`];
+}
+
+export function dropTable(tableName: string, isClustered: boolean) {
+  if (isClustered) {
+    return `DROP TABLE IF EXISTS ${tableName} ON CLUSTER '{cluster}'`;
+  }
+
+  return `DROP TABLE IF EXISTS ${tableName}`;
+}
+
+export function moveDataBetweenTables({
+  from,
+  to,
+  batch,
+}: {
+  from: string;
+  to: string;
+  batch?: {
+    column: string;
+    interval?: 'day' | 'week' | 'month';
+    transform?: (date: Date) => string;
+    endDate?: Date;
+    startDate?: Date;
+  };
+}): string[] {
+  const sqls: string[] = [];
+
+  if (!batch) {
+    return [`INSERT INTO ${to} SELECT * FROM ${from}`];
+  }
+
+  // Start from today and go back 3 years
+  const endDate = batch.endDate || new Date();
+  if (!batch.endDate) {
+    endDate.setDate(endDate.getDate() + 1); // Add 1 day to include today
+  }
+  const startDate = batch.startDate || new Date();
+  if (!batch.startDate) {
+    startDate.setFullYear(startDate.getFullYear() - 3);
+  }
+
+  let currentDate = endDate;
+  const interval = batch.interval || 'day';
+
+  while (currentDate > startDate) {
+    const previousDate = new Date(currentDate);
+
+    switch (interval) {
+      case 'month':
+        previousDate.setMonth(previousDate.getMonth() - 1);
+        break;
+      case 'week':
+        previousDate.setDate(previousDate.getDate() - 7);
+        // Ensure we don't go below startDate
+        if (previousDate < startDate) {
+          previousDate.setTime(startDate.getTime());
+        }
+        break;
+      // day
+      default:
+        previousDate.setDate(previousDate.getDate() - 1);
+        break;
+    }
+
+    const sql = `INSERT INTO ${to} 
+      SELECT * FROM ${from} 
+      WHERE ${batch.column} > '${batch.transform ? batch.transform(previousDate) : formatClickhouseDate(previousDate, true)}' 
+      AND ${batch.column} <= '${batch.transform ? batch.transform(currentDate) : formatClickhouseDate(currentDate, true)}'`;
+    sqls.push(sql);
+
+    currentDate = previousDate;
+  }
+
+  return sqls;
+}
+
+export function createMaterializedView({
+  name: tableName,
+  query,
+  engine = 'AggregatingMergeTree()',
+  orderBy,
+  partitionBy,
+  settings = {},
+  populate = false,
+  distributionHash = 'rand()',
+  replicatedVersion,
+  isClustered,
+}: CreateMaterializedViewOptions): string[] {
+  const settingsClause = Object.entries(settings).length
+    ? `SETTINGS ${Object.entries(settings)
+        .map(([key, value]) => `${key} = ${value}`)
+        .join(', ')}`
+    : '';
+
+  const partitionByClause = partitionBy ? `PARTITION BY ${partitionBy}` : '';
+
+  // Transform query to use replicated table names in clustered mode
+  const transformedQuery = query.replace(/\{(\w+)\}/g, (_, tableName) =>
+    isClustered ? replicated(tableName) : tableName,
+  );
+
+  if (!isClustered) {
+    return [
+      `CREATE MATERIALIZED VIEW IF NOT EXISTS ${tableName}
+ENGINE = ${engine}
+${partitionByClause}
+ORDER BY (${orderBy.join(', ')})
+${settingsClause}
+${populate ? 'POPULATE' : ''}
+AS ${transformedQuery}`.trim(),
+    ];
+  }
+
+  return [
+    // Replicated materialized view
+    `CREATE MATERIALIZED VIEW IF NOT EXISTS ${replicated(tableName)} ON CLUSTER '{cluster}'
+ENGINE = Replicated${engine.replace(/^(.+?)\((.+?)?\)/, `$1('${CLUSTER_REPLICA_PATH.replace('{replicatedVersion}', replicatedVersion)}', '{replica}', $2)`).replace(/, \)$/, ')')}
+${partitionByClause}
+ORDER BY (${orderBy.join(', ')})
+${settingsClause}
+${populate ? 'POPULATE' : ''}
+AS ${transformedQuery}`.trim(),
+    // Distributed materialized view
+    `CREATE TABLE IF NOT EXISTS ${tableName} ON CLUSTER '{cluster}' AS ${replicated(tableName)}
+ENGINE = Distributed('{cluster}', currentDatabase(), ${replicated(tableName)}, ${distributionHash})`,
+  ];
+}
+
+export function countRows(tableName: string) {
+  return `SELECT count() FROM ${tableName}`;
+}
+
+export async function runClickhouseMigrationCommands(sqls: string[]) {
+  let abort: AbortController | undefined;
+  let activeQueryId: string | undefined;
+
+  const handleTermination = async (signal: string) => {
+    console.warn(
+      `Received ${signal}. Cleaning up active queries before exit...`,
+    );
+
+    if (abort) {
+      abort.abort();
+    }
+  };
+
+  // Create bound handler functions
+  const handleSigterm = () => handleTermination('SIGTERM');
+  const handleSigint = () => handleTermination('SIGINT');
+
+  // Register handlers
+  process.on('SIGTERM', handleSigterm);
+  process.on('SIGINT', handleSigint);
+
+  try {
+    for (const sql of sqls) {
+      abort = new AbortController();
+      let timer: NodeJS.Timeout | undefined;
+      let resolve: ((value: unknown) => void) | undefined;
+      activeQueryId = crypto.createHash('sha256').update(sql).digest('hex');
+
+      console.log('----------------------------------------');
+      console.log('---| Running query | Query ID:', activeQueryId);
+      console.log('---| SQL |------------------------------');
+      console.log(sql);
+      console.log('----------------------------------------');
+
+      try {
+        const res = await Promise.race([
+          chMigrationClient.command({
+            query: sql,
+            query_id: activeQueryId,
+            abort_signal: abort?.signal,
+          }),
+          new Promise((r) => {
+            resolve = r;
+            let checking = false; // Add flag to prevent multiple concurrent checks
+
+            async function check() {
+              if (checking) return; // Skip if already checking
+              checking = true;
+
+              try {
+                const res = await chMigrationClient
+                  .query({
+                    query: `SELECT
+                              query_id,
+                              elapsed,
+                              read_rows,
+                              written_rows,
+                              memory_usage
+                            FROM system.processes 
+                            WHERE query_id = '${activeQueryId}'`,
+                    format: 'JSONEachRow',
+                  })
+                  .then((res) => res.json());
+
+                const formatMemory = (bytes: number) => {
+                  const units = ['B', 'KB', 'MB', 'GB'];
+                  let size = bytes;
+                  let unitIndex = 0;
+                  while (size >= 1024 && unitIndex < units.length - 1) {
+                    size /= 1024;
+                    unitIndex++;
+                  }
+                  return `${Math.round(size * 100) / 100}${units[unitIndex]}`;
+                };
+
+                const formatNumber = (num: number) => {
+                  return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
+                };
+
+                if (Array.isArray(res) && res.length > 0) {
+                  const { elapsed, read_rows, written_rows, memory_usage } =
+                    res[0] as any;
+                  console.log(
+                    `Progress: ${elapsed.toFixed(2)}s | Memory: ${formatMemory(memory_usage)} | Read: ${formatNumber(read_rows)} rows | Written: ${formatNumber(written_rows)} rows`,
+                  );
+                }
+              } finally {
+                checking = false;
+              }
+
+              timer = setTimeout(check, 5000); // Schedule next check after current one completes
+            }
+
+            // Start the first check after 5 seconds
+            timer = setTimeout(check, 5000);
+          }),
+        ]);
+
+        if (timer) {
+          clearTimeout(timer);
+        }
+        if (resolve) {
+          resolve(res);
+        }
+      } catch (e) {
+        console.log('Failed on query', sql);
+        throw e;
+      }
+    }
+  } catch (e) {
+    if (abort) {
+      abort.abort();
+    }
+
+    if (activeQueryId) {
+      try {
+        await chMigrationClient.command({
+          query: `KILL QUERY WHERE query_id = '${activeQueryId}'`,
+        });
+        console.log(`Successfully killed query ${activeQueryId}`);
+      } catch (err) {
+        console.error(`Failed to kill query ${activeQueryId}:`, err);
+      }
+    }
+
+    throw e;
+  } finally {
+    // Clean up event listeners
+    process.off('SIGTERM', handleSigterm);
+    process.off('SIGINT', handleSigint);
+  }
+}
--- a/packages/db/src/services/chart.service.ts
+++ b/packages/db/src/services/chart.service.ts
@@ -13,7 +13,7 @@ import {
  TABLE_NAMES,
  formatClickhouseDate,
  toDate,
-} from '../clickhouse-client';
+} from '../clickhouse/client';
 import { createSqlBuilder } from '../sql-builder';

 export function transformPropertyKey(property: string) {
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -12,7 +12,7 @@ import {
  chQuery,
  convertClickhouseDateToJs,
  formatClickhouseDate,
-} from '../clickhouse-client';
+} from '../clickhouse/client';
 import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 import { createSqlBuilder } from '../sql-builder';
--- a/packages/db/src/services/organization.service.ts
+++ b/packages/db/src/services/organization.service.ts
@@ -143,6 +143,10 @@ export async function connectUserToOrganization({
    throw new Error('Invite not found');
  }

+  if (process.env.ALLOW_INVITATION === 'false') {
+    throw new Error('Invitations are not allowed');
+  }
+
  if (invite.expiresAt < new Date()) {
    throw new Error('Invite expired');
  }
--- a/packages/db/src/services/profile.service.ts
+++ b/packages/db/src/services/profile.service.ts
@@ -11,7 +11,7 @@ import {
  ch,
  chQuery,
  formatClickhouseDate,
-} from '../clickhouse-client';
+} from '../clickhouse/client';
 import { createSqlBuilder } from '../sql-builder';

 export type IProfileMetrics = {
--- a/packages/db/src/services/retention.service.ts
+++ b/packages/db/src/services/retention.service.ts
@@ -1,6 +1,6 @@
 import { escape } from 'sqlstring';

-import { TABLE_NAMES, chQuery } from '../clickhouse-client';
+import { TABLE_NAMES, chQuery } from '../clickhouse/client';

 type IGetWeekRetentionInput = {
  projectId: string;
--- a/packages/db/src/sql-builder.ts
+++ b/packages/db/src/sql-builder.ts
@@ -1,4 +1,4 @@
-import { TABLE_NAMES } from './clickhouse-client';
+import { TABLE_NAMES } from './clickhouse/client';

 export interface SqlBuilderObject {
  where: Record<string, string>;
--- a/packages/email/src/index.tsx
+++ b/packages/email/src/index.tsx
@@ -13,11 +13,6 @@ export async function sendEmail<T extends TemplateKey>(
    data: z.infer<Templates[T]['schema']>;
  },
 ) {
-  if (!process.env.RESEND_API_KEY) {
-    return null;
-  }
-
-  const resend = new Resend(process.env.RESEND_API_KEY);
  const { to, data } = options;
  const { subject, Component, schema } = templates[template];
  const props = schema.safeParse(data);
@@ -27,6 +22,14 @@ export async function sendEmail<T extends TemplateKey>(
    return null;
  }

+  if (!process.env.RESEND_API_KEY) {
+    console.log('No RESEND_API_KEY found, here is the data');
+    console.log(data);
+    return null;
+  }
+
+  const resend = new Resend(process.env.RESEND_API_KEY);
+
  try {
    const res = await resend.emails.send({
      from: FROM,
--- a/packages/trpc/src/routers/auth.ts
+++ b/packages/trpc/src/routers/auth.ts
@@ -37,6 +37,38 @@ import {

 const zProvider = z.enum(['email', 'google', 'github']);

+async function getIsRegistrationAllowed(inviteId?: string | null) {
+  // ALLOW_REGISTRATION is always undefined in cloud
+  if (process.env.ALLOW_REGISTRATION === undefined) {
+    return true;
+  }
+
+  // Self-hosting logic
+  // 1. First user is always allowed
+  const count = await db.user.count();
+  if (count === 0) {
+    return true;
+  }
+
+  // 2. If there is an invite, check if it is valid
+  if (inviteId) {
+    if (process.env.ALLOW_INVITATION === 'false') {
+      return false;
+    }
+
+    const invite = await db.invite.findUnique({
+      where: {
+        id: inviteId,
+      },
+    });
+
+    return !!invite;
+  }
+
+  // 3. Otherwise, check if general registration is allowed
+  return process.env.ALLOW_REGISTRATION !== 'false';
+}
+
 export const authRouter = createTRPCRouter({
  signOut: publicProcedure.mutation(async ({ ctx }) => {
    deleteSessionTokenCookie(ctx.setCookie);
@@ -46,7 +78,15 @@ export const authRouter = createTRPCRouter({
  }),
  signInOAuth: publicProcedure
    .input(z.object({ provider: zProvider, inviteId: z.string().nullish() }))
-    .mutation(({ input, ctx }) => {
+    .mutation(async ({ input, ctx }) => {
+      const isRegistrationAllowed = await getIsRegistrationAllowed(
+        input.inviteId,
+      );
+
+      if (!isRegistrationAllowed) {
+        throw TRPCAccessError('Registrations are not allowed');
+      }
+
      const { provider } = input;

      if (input.inviteId) {
@@ -95,6 +135,14 @@ export const authRouter = createTRPCRouter({
  signUpEmail: publicProcedure
    .input(zSignUpEmail)
    .mutation(async ({ input, ctx }) => {
+      const isRegistrationAllowed = await getIsRegistrationAllowed(
+        input.inviteId,
+      );
+
+      if (!isRegistrationAllowed) {
+        throw TRPCAccessError('Registrations are not allowed');
+      }
+
      const provider = 'email';
      const user = await getUserAccount({
        email: input.email,