feat(root): added migrations and optimized profile table

2024-09-10 10:08:26 +02:00
parent 2258fed24a
commit b44f1958a2
22 changed files with 280 additions and 169 deletions
--- a/self-hosting/.env.template
+++ b/self-hosting/.env.template
@@ -10,9 +10,6 @@ BATCH_INTERVAL="10000"
 # Will be replaced with the setup script
 REDIS_URL="$REDIS_URL"
 CLICKHOUSE_URL="$CLICKHOUSE_URL"
-CLICKHOUSE_DB="$CLICKHOUSE_DB"
-CLICKHOUSE_USER="$CLICKHOUSE_USER"
-CLICKHOUSE_PASSWORD="$CLICKHOUSE_PASSWORD"
 DATABASE_URL="$DATABASE_URL"
 DATABASE_URL_DIRECT="$DATABASE_URL_DIRECT"
 NEXT_PUBLIC_DASHBOARD_URL="$NEXT_PUBLIC_DASHBOARD_URL"
--- a/self-hosting/clickhouse/init-db.sh
+++ b/self-hosting/clickhouse/init-db.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+clickhouse client -n <<-EOSQL
+    CREATE DATABASE IF NOT EXISTS openpanel;
+EOSQL
--- a/self-hosting/docker-compose.template.yml
+++ b/self-hosting/docker-compose.template.yml
@@ -20,50 +20,41 @@ services:
    restart: always
    volumes:
      - op-db-data:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_PASSWORD
    healthcheck:
      test: ['CMD-SHELL', 'pg_isready -U postgres']
      interval: 10s
      timeout: 5s
      retries: 5
-    ports:
-      - 5431:5432
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+    # Uncomment to expose ports
+    # ports:
+    #   - 5432:5432

  op-kv:
    image: redis:7.2.5-alpine
    restart: always
    volumes:
      - op-kv-data:/data
-    command:
-      [
-        'redis-server',
-        '--requirepass',
-        '${REDIS_PASSWORD}',
-        '--maxmemory-policy',
-        'noeviction',
-      ]
-    ports:
-      - 6378:6379
-    environment:
-      - REDIS_PASSWORD=${REDIS_PASSWORD}
+    command: ['redis-server', '--maxmemory-policy', 'noeviction']
+    # Uncomment to expose ports
+    # ports:
+    #   - 6379:6379

  op-geo:
    image: observabilitystack/geoip-api:latest
    restart: always

  op-ch:
-    image: clickhouse/clickhouse-server:23.3.7.5-alpine
+    image: clickhouse/clickhouse-server:24.3.2-alpine
    restart: always
    volumes:
      - op-ch-data:/var/lib/clickhouse
      - op-ch-logs:/var/log/clickhouse-server
      - ./clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/op-config.xml:ro
      - ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro
-    environment:
-      - CLICKHOUSE_DB
-      - CLICKHOUSE_USER
-      - CLICKHOUSE_PASSWORD
+      - ./clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
    healthcheck:
      test: ['CMD-SHELL', 'clickhouse-client --query "SELECT 1"']
      interval: 10s
@@ -73,37 +64,34 @@ services:
      nofile:
        soft: 262144
        hard: 262144
-    ports:
-      - 8999:9000
-      - 8122:8123
-
-  op-ch-migrator:
-    image: clickhouse/clickhouse-server:23.3.7.5-alpine
-    depends_on:
-      - op-ch
-    volumes:
-      - ../packages/db/clickhouse_init.sql:/migrations/clickhouse_init.sql
-    environment:
-      - CLICKHOUSE_DB
-      - CLICKHOUSE_USER
-      - CLICKHOUSE_PASSWORD
-    entrypoint: /bin/sh -c
-    command: >
-      "
-      echo 'Waiting for ClickHouse to start...';
-      while ! clickhouse-client --host op-ch --user=$CLICKHOUSE_USER --password=$CLICKHOUSE_PASSWORD --query 'SELECT 1;' 2>/dev/null; do
-        echo 'ClickHouse is unavailable - sleeping 1s...';
-        sleep 1;
-      done;
-
-      echo 'ClickHouse started. Running migrations...';
-      clickhouse-client --host op-ch --database=$CLICKHOUSE_DB --user=$CLICKHOUSE_USER --password=$CLICKHOUSE_PASSWORD --queries-file /migrations/clickhouse_init.sql;
-      "
+    # Uncomment to expose ports
+    # ports:
+    #   - 9000:9000
+    #   - 8123:8123

  op-api:
    image: lindesvard/openpanel-api:latest
    restart: always
-    command: sh -c "sleep 10 && pnpm -r run migrate:deploy && pnpm start"
+    command: >
+      sh -c "
+        echo 'Waiting for PostgreSQL to be ready...'
+        while ! nc -z op-db 5432; do
+          sleep 1
+        done
+        echo 'PostgreSQL is ready'
+
+        # Add wait for ClickHouse
+        echo 'Waiting for ClickHouse to be ready...'
+        while ! nc -z op-ch 8123; do
+          sleep 1
+        done
+        echo 'ClickHouse is ready'
+
+        echo 'Running migrations...'
+        pnpm -r run migrate:deploy
+
+        pnpm start
+      "
    depends_on:
      - op-db
      - op-ch
@@ -116,9 +104,7 @@ services:
    image: lindesvard/openpanel-dashboard:latest
    restart: always
    depends_on:
-      - op-db
-      - op-ch
-      - op-kv
+      - op-api
    env_file:
      - .env

@@ -126,9 +112,7 @@ services:
    image: lindesvard/openpanel-worker:latest
    restart: always
    depends_on:
-      - op-db
-      - op-ch
-      - op-kv
+      - op-api
    env_file:
      - .env
    deploy:
--- a/self-hosting/quiz.ts
+++ b/self-hosting/quiz.ts
@@ -112,12 +112,7 @@ function removeServiceFromDockerCompose(serviceName: string) {
 }

 function writeEnvFile(envs: {
-  POSTGRES_PASSWORD: string | undefined;
-  REDIS_PASSWORD: string | undefined;
  CLICKHOUSE_URL: string;
-  CLICKHOUSE_DB: string;
-  CLICKHOUSE_USER: string;
-  CLICKHOUSE_PASSWORD: string;
  REDIS_URL: string;
  DATABASE_URL: string;
  DOMAIN_NAME: string;
@@ -131,9 +126,6 @@ function writeEnvFile(envs: {

  let newEnvFile = envTemplate
    .replace('$CLICKHOUSE_URL', envs.CLICKHOUSE_URL)
-    .replace('$CLICKHOUSE_DB', envs.CLICKHOUSE_DB)
-    .replace('$CLICKHOUSE_USER', envs.CLICKHOUSE_USER)
-    .replace('$CLICKHOUSE_PASSWORD', envs.CLICKHOUSE_PASSWORD)
    .replace('$REDIS_URL', envs.REDIS_URL)
    .replace('$DATABASE_URL', envs.DATABASE_URL)
    .replace('$DATABASE_URL_DIRECT', envs.DATABASE_URL)
@@ -149,10 +141,6 @@ function writeEnvFile(envs: {
    .replace('$CLERK_SECRET_KEY', envs.CLERK_SECRET_KEY)
    .replace('$CLERK_SIGNING_SECRET', envs.CLERK_SIGNING_SECRET);

-  if (envs.POSTGRES_PASSWORD) {
-    newEnvFile += `\nPOSTGRES_PASSWORD=${envs.POSTGRES_PASSWORD}`;
-  }
-
  fs.writeFileSync(
    envPath,
    newEnvFile
@@ -234,26 +222,9 @@ async function initiateOnboarding() {
      {
        type: 'input',
        name: 'CLICKHOUSE_URL',
-        message: 'Enter your ClickHouse URL:',
-        default: process.env.DEBUG ? 'http://clickhouse:8123' : undefined,
-      },
-      {
-        type: 'input',
-        name: 'CLICKHOUSE_DB',
-        message: 'Enter your ClickHouse DB name:',
-        default: process.env.DEBUG ? 'db_openpanel' : undefined,
-      },
-      {
-        type: 'input',
-        name: 'CLICKHOUSE_USER',
-        message: 'Enter your ClickHouse user name:',
-        default: process.env.DEBUG ? 'user_openpanel' : undefined,
-      },
-      {
-        type: 'input',
-        name: 'CLICKHOUSE_PASSWORD',
-        message: 'Enter your ClickHouse password:',
-        default: process.env.DEBUG ? 'ch_password' : undefined,
+        message:
+          'Enter your ClickHouse URL (format: http://user:pw@host:port/db):',
+        default: process.env.DEBUG ? 'http://op-ch:8123/openpanel' : undefined,
      },
    ]);

@@ -268,8 +239,8 @@ async function initiateOnboarding() {
      {
        type: 'input',
        name: 'REDIS_URL',
-        message: 'Enter your Redis URL:',
-        default: process.env.DEBUG ? 'redis://redis:6379' : undefined,
+        message: 'Enter your Redis URL (format: redis://user:pw@host:port/db):',
+        default: process.env.DEBUG ? 'redis://op-kv:6379' : undefined,
      },
    ]);
    envs = {
@@ -283,9 +254,10 @@ async function initiateOnboarding() {
      {
        type: 'input',
        name: 'DATABASE_URL',
-        message: 'Enter your Database URL:',
+        message:
+          'Enter your Database URL (format: postgresql://user:pw@host:port/db):',
        default: process.env.DEBUG
-          ? 'postgresql://postgres:postgres@postgres:5432/postgres?schema=public'
+          ? 'postgresql://postgres:postgres@op-db:5432/postgres?schema=public'
          : undefined,
      },
    ]);
@@ -399,20 +371,13 @@ async function initiateOnboarding() {

  console.log('');
  console.log('Creating .env file...\n');
-  const POSTGRES_PASSWORD = generatePassword(20);
-  const REDIS_PASSWORD = generatePassword(20);

  writeEnvFile({
-    POSTGRES_PASSWORD: envs.DATABASE_URL ? undefined : POSTGRES_PASSWORD,
-    REDIS_PASSWORD: envs.REDIS_URL ? undefined : REDIS_PASSWORD,
-    CLICKHOUSE_URL: envs.CLICKHOUSE_URL || 'http://op-ch:8123',
-    CLICKHOUSE_DB: envs.CLICKHOUSE_DB || 'openpanel',
-    CLICKHOUSE_USER: envs.CLICKHOUSE_USER || 'openpanel',
-    CLICKHOUSE_PASSWORD: envs.CLICKHOUSE_PASSWORD || generatePassword(20),
+    CLICKHOUSE_URL: envs.CLICKHOUSE_URL || 'http://op-ch:8123/openpanel',
    REDIS_URL: envs.REDIS_URL || 'redis://op-kv:6379',
    DATABASE_URL:
      envs.DATABASE_URL ||
-      `postgresql://postgres:${POSTGRES_PASSWORD}@op-db:5432/postgres?schema=public`,
+      `postgresql://postgres:postgres@op-db:5432/postgres?schema=public`,
    DOMAIN_NAME: domainNameResponse.domainName,
    NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:
      clerkResponse.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY || '',