feat(root): added migrations and optimized profile table

packages/db/migrations/20240906185616_init.sql

@@ -1,6 +1,6 @@
-CREATE DATABASE IF NOT EXISTS openpanel;
-
-CREATE TABLE IF NOT EXISTS openpanel.self_hosting
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS self_hosting
 (
     created_at Date,
     domain String,
@@ -9,9 +9,10 @@ CREATE TABLE IF NOT EXISTS openpanel.self_hosting
 ENGINE = MergeTree()
 ORDER BY (domain, created_at)
 PARTITION BY toYYYYMM(created_at);
+-- +goose StatementEnd
 
-
-CREATE TABLE IF NOT EXISTS openpanel.events_v2 (
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS events_v2 (
   `id` UUID DEFAULT generateUUIDv4(),
   `name` String,
   `sdk_name` String,
@@ -48,20 +49,25 @@ CREATE TABLE IF NOT EXISTS openpanel.events_v2 (
 ) ENGINE = MergeTree PARTITION BY toYYYYMM(created_at)
 ORDER BY
   (project_id, toDate(created_at), profile_id, name) SETTINGS index_granularity = 8192;
+-- +goose StatementEnd
 
-CREATE TABLE IF NOT EXISTS openpanel.events_bots (
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS events_bots (
   `id` UUID DEFAULT generateUUIDv4(),
   `project_id` String,
   `name` String,
   `type` String,
   `path` String,
-  `created_at` DateTime64(3),
+  `created_at` DateTime64(3)
 ) ENGINE MergeTree
 ORDER BY
   (project_id, created_at) SETTINGS index_granularity = 8192;
+-- +goose StatementEnd
 
-CREATE TABLE IF NOT EXISTS openpanel.profiles (
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS profiles (
   `id` String,
+  `is_external` Bool,
   `first_name` String,
   `last_name` String,
   `email` String,
@@ -72,8 +78,10 @@ CREATE TABLE IF NOT EXISTS openpanel.profiles (
 ) ENGINE = ReplacingMergeTree(created_at)
 ORDER BY
   (id) SETTINGS index_granularity = 8192;
+-- +goose StatementEnd
 
-CREATE TABLE IF NOT EXISTS openpanel.profile_aliases (
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS profile_aliases (
   `project_id` String,
   `profile_id` String,
   `alias` String,
@@ -81,8 +89,9 @@ CREATE TABLE IF NOT EXISTS openpanel.profile_aliases (
 ) ENGINE = MergeTree
 ORDER BY
   (project_id, profile_id, alias, created_at) SETTINGS index_granularity = 8192;
+-- +goose StatementEnd
 
---- Materialized views (DAU)
+-- +goose StatementBegin
 CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMMDD(date)
 ORDER BY
   (project_id, date) POPULATE AS
@@ -94,4 +103,10 @@ FROM
   events_v2
 GROUP BY
   date,
-  project_id;
+  project_id;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query';
+-- +goose StatementEnd

packages/db/migrations/20240907202846_optimize_profiles.sql

@@ -0,0 +1,44 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE profiles_tmp
+(
+    `id` String,
+    `is_external` Bool,
+    `first_name` String,
+    `last_name` String,
+    `email` String,
+    `avatar` String,
+    `properties` Map(String, String),
+    `project_id` String,
+    `created_at` DateTime,
+    INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
+    INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
+    INDEX idx_email email TYPE bloom_filter GRANULARITY 1
+)
+ENGINE = ReplacingMergeTree(created_at)
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (project_id, created_at, id)
+SETTINGS index_granularity = 8192;
+-- +goose StatementEnd
+-- +goose StatementBegin
+INSERT INTO profiles_tmp SELECT
+    id,
+    is_external,
+    first_name,
+    last_name,
+    email,
+    avatar,
+    properties,
+    project_id,
+    created_at
+FROM profiles;
+-- +goose StatementEnd
+-- +goose StatementBegin
+OPTIMIZE TABLE profiles_tmp FINAL;
+-- +goose StatementEnd
+-- +goose StatementBegin
+RENAME TABLE profiles TO profiles_old, profiles_tmp TO profiles;
+-- +goose StatementEnd
+-- +goose StatementBegin
+DROP TABLE profiles_old;
+-- +goose StatementEnd

packages/db/migrations/goose

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+
+if [ -z "$CLICKHOUSE_URL" ]; then
+  echo "CLICKHOUSE_URL is not set"
+  exit 1
+fi
+
+export GOOSE_DBSTRING=$CLICKHOUSE_URL
+
+goose clickhouse --dir ./migrations $@

packages/db/package.json

@@ -3,9 +3,12 @@
   "version": "0.0.1",
   "main": "index.ts",
   "scripts": {
+    "goose": "pnpm with-env ./migrations/goose",
     "codegen": "pnpm with-env prisma generate",
     "migrate": "pnpm with-env prisma migrate dev",
-    "migrate:deploy": "pnpm with-env prisma migrate deploy",
+    "migrate:deploy:db": "pnpm with-env prisma migrate deploy",
+    "migrate:deploy:ch": "pnpm goose up",
+    "migrate:deploy": "pnpm migrate:deploy:db && pnpm migrate:deploy:ch",
     "lint": "eslint .",
     "format": "prettier --check \"**/*.{mjs,ts,md,json}\"",
     "typecheck": "tsc --noEmit",
@@ -44,4 +47,4 @@
     ]
   },
   "prettier": "@openpanel/prettier-config"
-}
+}

packages/db/src/clickhouse-client.ts

@@ -9,13 +9,12 @@ export const TABLE_NAMES = {
   profiles: 'profiles',
   alias: 'profile_aliases',
   self_hosting: 'self_hosting',
+  events_bots: 'events_bots',
+  dau_mv: 'dau_mv',
 };
 
 export const originalCh = createClient({
   url: process.env.CLICKHOUSE_URL,
-  username: process.env.CLICKHOUSE_USER,
-  password: process.env.CLICKHOUSE_PASSWORD,
-  database: process.env.CLICKHOUSE_DB,
   max_open_connections: 30,
   request_timeout: 30000,
   keep_alive: {

packages/db/src/services/event.service.ts

@@ -225,23 +225,24 @@ export async function getEvents(
   options: GetEventsOptions = {}
 ): Promise<IServiceEvent[]> {
   const events = await chQuery<IClickhouseEvent>(sql);
-  if (options.profile) {
+  const projectId = events[0]?.project_id;
+  if (options.profile && projectId) {
     const ids = events.map((e) => e.profile_id);
-    const profiles = await getProfiles(ids);
+    const profiles = await getProfiles(ids, projectId);
 
     for (const event of events) {
       event.profile = profiles.find((p) => p.id === event.profile_id);
     }
   }
 
-  if (options.meta) {
+  if (options.meta && projectId) {
     const names = uniq(events.map((e) => e.name));
     const metas = await db.eventMeta.findMany({
       where: {
         name: {
           in: names,
         },
-        projectId: events[0]?.project_id,
+        projectId,
       },
       select: options.meta === true ? undefined : options.meta,
     });

packages/db/src/services/profile.service.ts

@@ -69,7 +69,7 @@ interface GetProfileListOptions {
   search?: string;
 }
 
-export async function getProfiles(ids: string[]) {
+export async function getProfiles(ids: string[], projectId: string) {
   const filteredIds = uniq(ids.filter((id) => id !== ''));
 
   if (filteredIds.length === 0) {
@@ -78,8 +78,10 @@ export async function getProfiles(ids: string[]) {
 
   const data = await chQuery<IClickhouseProfile>(
     `SELECT id, first_name, last_name, email, avatar, is_external
-    FROM profiles FINAL 
-    WHERE id IN (${filteredIds.map((id) => escape(id)).join(',')})
+    FROM ${TABLE_NAMES.profiles} FINAL 
+    WHERE 
+      project_id = ${escape(projectId)} AND
+      id IN (${filteredIds.map((id) => escape(id)).join(',')})
     `
   );
 
@@ -94,18 +96,14 @@ export async function getProfileList({
   search,
 }: GetProfileListOptions) {
   const { sb, getSql } = createSqlBuilder();
-  sb.from = 'profiles FINAL';
+  sb.from = `${TABLE_NAMES.profiles} FINAL`;
   sb.select.all = '*';
   sb.where.project_id = `project_id = ${escape(projectId)}`;
   sb.limit = take;
   sb.offset = Math.max(0, (cursor ?? 0) * take);
   sb.orderBy.created_at = 'created_at DESC';
   if (search) {
-    if (search.includes('@')) {
-      sb.where.email = `email ILIKE '%${search}%'`;
-    } else {
-      sb.where.first_name = `first_name ILIKE '%${search}%' OR last_name ILIKE '%${search}%'`;
-    }
+    sb.where.search = `(email ILIKE '%${search}%' OR first_name ILIKE '%${search}%' OR last_name ILIKE '%${search}%')`;
   }
   const data = await chQuery<IClickhouseProfile>(getSql());
   return data.map(transformProfile);

packages/db/src/services/retention.service.ts

@@ -121,7 +121,7 @@ export function getRollingActiveUsers({
       FROM
       (
           SELECT *
-          FROM dau_mv
+          FROM ${TABLE_NAMES.dau_mv}
           WHERE project_id = ${escape(projectId)}
       )
       ARRAY JOIN range(${days}) AS n

packages/trpc/src/routers/chart.helpers.ts

@@ -430,7 +430,10 @@ export async function getFunnelStep({
     id: string;
   }>(profileIdsQuery);
 
-  return getProfiles(res.map((r) => r.id));
+  return getProfiles(
+    res.map((r) => r.id),
+    projectId
+  );
 }
 
 export async function getChartSerie(payload: IGetChartDataInput) {

packages/trpc/src/routers/event.ts

@@ -151,12 +151,12 @@ export const eventRouter = createTRPCRouter({
           path: string;
           created_at: string;
         }>(
-          `SELECT * FROM events_bots WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit} OFFSET ${(cursor ?? 0) * limit}`
+          `SELECT * FROM ${TABLE_NAMES.events_bots} WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit} OFFSET ${(cursor ?? 0) * limit}`
         ),
         chQuery<{
           count: number;
         }>(
-          `SELECT count(*) as count FROM events_bots WHERE project_id = ${escape(projectId)}`
+          `SELECT count(*) as count FROM ${TABLE_NAMES.events_bots} WHERE project_id = ${escape(projectId)}`
         ),
       ]);
 

packages/trpc/src/routers/profile.ts

@@ -17,7 +17,7 @@ export const profileRouter = createTRPCRouter({
     .input(z.object({ projectId: z.string() }))
     .query(async ({ input: { projectId } }) => {
       const events = await chQuery<{ keys: string[] }>(
-        `SELECT distinct mapKeys(properties) as keys from profiles where project_id = ${escape(projectId)};`
+        `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.profiles} where project_id = ${escape(projectId)};`
       );
 
       const properties = events
@@ -61,7 +61,10 @@ export const profileRouter = createTRPCRouter({
       const res = await chQuery<{ profile_id: string; count: number }>(
         `SELECT profile_id, count(*) as count from ${TABLE_NAMES.events} where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT ${take} ${cursor ? `OFFSET ${cursor * take}` : ''}`
       );
-      const profiles = await getProfiles(res.map((r) => r.profile_id));
+      const profiles = await getProfiles(
+        res.map((r) => r.profile_id),
+        projectId
+      );
       return (
         res
           .map((item) => {
@@ -84,7 +87,7 @@ export const profileRouter = createTRPCRouter({
     )
     .query(async ({ input: { property, projectId } }) => {
       const { sb, getSql } = createSqlBuilder();
-      sb.from = 'profiles';
+      sb.from = TABLE_NAMES.profiles;
       sb.where.project_id = `project_id = ${escape(projectId)}`;
       if (property.startsWith('properties.')) {
         sb.select.values = `distinct arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(