fix(db): merging profiles

packages/db/migrations/20240917184138_fix_profile_merges.sql

@@ -0,0 +1,55 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE profiles_fixed
+(
+    `id` String,
+    `is_external` Bool,
+    `first_name` String,
+    `last_name` String,
+    `email` String,
+    `avatar` String,
+    `properties` Map(String, String),
+    `project_id` String,
+    `created_at` DateTime,
+    INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
+    INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
+    INDEX idx_email email TYPE bloom_filter GRANULARITY 1
+)
+ENGINE = ReplacingMergeTree(created_at)
+PARTITION BY toYYYYMM(created_at)
+ORDER BY (project_id, id)
+SETTINGS index_granularity = 8192;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+INSERT INTO profiles_fixed SELECT
+    id,
+    is_external,
+    first_name,
+    last_name,
+    email,
+    avatar,
+    properties,
+    project_id,
+    created_at
+FROM profiles;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+OPTIMIZE TABLE profiles_fixed FINAL;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+RENAME TABLE profiles TO profiles_old, profiles_fixed TO profiles;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+DROP TABLE profiles_old;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+-- This is a destructive migration, so the down migration is not provided.
+-- If needed, you should restore from a backup.
+SELECT 'down migration not implemented';
+-- +goose StatementEnd

packages/db/migrations/goose

@@ -8,4 +8,20 @@ fi
 
 export GOOSE_DBSTRING=$CLICKHOUSE_URL
 
+echo "Clickhouse migration script"
+echo ""
+echo "================="
+echo "Selected database: $GOOSE_DBSTRING"
+echo "================="
+echo ""
+if [ "$1" != "create" ] && [ -z "$CI" ]; then
+  read -p "Are you sure you want to run migrations on this database? (y/n) " -n 1 -r
+  echo
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Migration cancelled."
+    exit 0
+  fi
+fi
+
+
 goose clickhouse --dir ./migrations $@

self-hosting/docker-compose.template.yml

@@ -21,7 +21,7 @@ services:
     volumes:
       - op-db-data:/var/lib/postgresql/data
     healthcheck:
-      test: ['CMD-SHELL', 'pg_isready -U postgres']
+      test: [ 'CMD-SHELL', 'pg_isready -U postgres' ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -37,7 +37,7 @@ services:
     restart: always
     volumes:
       - op-kv-data:/data
-    command: ['redis-server', '--maxmemory-policy', 'noeviction']
+    command: [ 'redis-server', '--maxmemory-policy', 'noeviction' ]
     # Uncomment to expose ports
     # ports:
     #   - 6379:6379
@@ -56,7 +56,7 @@ services:
       - ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro
       - ./clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
     healthcheck:
-      test: ['CMD-SHELL', 'clickhouse-client --query "SELECT 1"']
+      test: [ 'CMD-SHELL', 'clickhouse-client --query "SELECT 1"' ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -88,7 +88,7 @@ services:
         echo 'ClickHouse is ready'
 
         echo 'Running migrations...'
-        pnpm -r run migrate:deploy
+        CI=true pnpm -r run migrate:deploy
 
         pnpm start
       "