feature(auth): replace clerk.com with custom auth (#103)

* feature(auth): replace clerk.com with custom auth * minor fixes * remove notification preferences * decrease live events interval fix(api): cookies.. # Conflicts: # .gitignore # apps/api/src/index.ts # apps/dashboard/src/app/providers.tsx # packages/trpc/src/trpc.ts
2024-12-18 21:30:39 +01:00
parent f28802b1c2
commit d31d9924a5
151 changed files with 18484 additions and 12853 deletions
--- a/self-hosting/.env.template
+++ b/self-hosting/.env.template
@@ -1,10 +1,6 @@
 NODE_ENV="production"
 SELF_HOSTED="true"
 GEO_IP_HOST="http://op-geo:8080"
-NEXT_PUBLIC_CLERK_SIGN_IN_URL="/login"
-NEXT_PUBLIC_CLERK_SIGN_UP_URL="/register"
-NEXT_PUBLIC_CLERK_AFTER_SIGN_IN_URL="/"
-NEXT_PUBLIC_CLERK_AFTER_SIGN_UP_URL="/"
 BATCH_SIZE="5000"
 BATCH_INTERVAL="10000"
 # Will be replaced with the setup script
@@ -14,6 +10,4 @@ DATABASE_URL="$DATABASE_URL"
 DATABASE_URL_DIRECT="$DATABASE_URL_DIRECT"
 NEXT_PUBLIC_DASHBOARD_URL="$NEXT_PUBLIC_DASHBOARD_URL"
 NEXT_PUBLIC_API_URL="$NEXT_PUBLIC_API_URL"
-NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY="$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"
-CLERK_SECRET_KEY="$CLERK_SECRET_KEY"
-CLERK_SIGNING_SECRET="$CLERK_SIGNING_SECRET"
+COOKIE_SECRET="$COOKIE_SECRET"
--- a/self-hosting/clickhouse/clickhouse-config.xml
+++ b/self-hosting/clickhouse/clickhouse-config.xml
@@ -22,4 +22,32 @@
    <session_log remove="remove"/>
    <part_log remove="remove"/>

+    <listen_host>0.0.0.0</listen_host>
+    <interserver_listen_host>0.0.0.0</interserver_listen_host>
+    <interserver_http_host>op-ch</interserver_http_host>
+
+    <macros>
+        <shard>1</shard>
+        <replica>replica1</replica>
+        <cluster>openpanel_cluster</cluster>
+    </macros>
+
+    <zookeeper>
+        <node index="1">
+            <host>op-zk</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+
+    <remote_servers>
+        <openpanel_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>op-ch</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </openpanel_cluster>
+    </remote_servers>
 </clickhouse>
--- a/self-hosting/clickhouse/clickhouse-keeper-config.xml
+++ b/self-hosting/clickhouse/clickhouse-keeper-config.xml
@@ -0,0 +1,44 @@
+<clickhouse>    
+    <logger>
+        <level>information</level>
+        <console>true</console>
+    </logger>
+    
+    <path>/var/lib/clickhouse/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+
+    <timezone>UTC</timezone>
+    <mlock_executable>false</mlock_executable>
+
+    <listen_host>0.0.0.0</listen_host>
+    <interserver_listen_host>0.0.0.0</interserver_listen_host>
+    <interserver_http_host>op-zk</interserver_http_host>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <listen_host>::</listen_host>
+        <interserver_listen_host>::</interserver_listen_host>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>op-zk</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+
+    <distributed_ddl>
+        <path>/clickhouse/production/task_queue/ddl</path>
+    </distributed_ddl>
+</clickhouse> 
--- a/self-hosting/docker-compose.template.yml
+++ b/self-hosting/docker-compose.template.yml
@@ -64,10 +64,18 @@ services:
      nofile:
        soft: 262144
        hard: 262144
-    # Uncomment to expose ports
-    # ports:
-    #   - 9000:9000
-    #   - 8123:8123
+
+  op-zk:
+    image: clickhouse/clickhouse-server:24.3.2-alpine
+    volumes:
+      - op-zk-data:/var/lib/clickhouse
+      - ./clickhouse/clickhouse-keeper-config.xml:/etc/clickhouse-server/config.xml
+    command: [ 'clickhouse-keeper', '--config-file', '/etc/clickhouse-server/config.xml' ]
+    restart: always
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144

  op-api:
    image: lindesvard/openpanel-api:latest
@@ -80,7 +88,6 @@ services:
        done
        echo 'PostgreSQL is ready'

-        # Add wait for ClickHouse
        echo 'Waiting for ClickHouse to be ready...'
        while ! nc -z op-ch 8123; do
          sleep 1
@@ -132,3 +139,5 @@ volumes:
    driver: local
  op-proxy-config:
    driver: local
+  op-zk-data:
+    driver: local
--- a/self-hosting/quiz.ts
+++ b/self-hosting/quiz.ts
@@ -116,15 +116,14 @@ function writeEnvFile(envs: {
  REDIS_URL: string;
  DATABASE_URL: string;
  DOMAIN_NAME: string;
-  NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: string;
-  CLERK_SECRET_KEY: string;
-  CLERK_SIGNING_SECRET: string;
+  COOKIE_SECRET: string;
 }) {
  const envTemplatePath = path.resolve(__dirname, '.env.template');
  const envPath = path.resolve(__dirname, '.env');
  const envTemplate = fs.readFileSync(envTemplatePath, 'utf-8');

  const newEnvFile = envTemplate
+    .replace('COOKIE_SECRET', envs.COOKIE_SECRET)
    .replace('$CLICKHOUSE_URL', envs.CLICKHOUSE_URL)
    .replace('$REDIS_URL', envs.REDIS_URL)
    .replace('$DATABASE_URL', envs.DATABASE_URL)
@@ -133,13 +132,7 @@ function writeEnvFile(envs: {
    .replace(
      '$NEXT_PUBLIC_API_URL',
      `${stripTrailingSlash(envs.DOMAIN_NAME)}/api`,
-    )
-    .replace(
-      '$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY',
-      envs.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY,
-    )
-    .replace('$CLERK_SECRET_KEY', envs.CLERK_SECRET_KEY)
-    .replace('$CLERK_SIGNING_SECRET', envs.CLERK_SIGNING_SECRET);
+    );

  fs.writeFileSync(
    envPath,
@@ -169,11 +162,7 @@ async function initiateOnboarding() {
    'Before you continue, please make sure you have the following:',
    `${T}1. Docker and Docker Compose installed on your machine.`,
    `${T}2. A domain name that you can use for this setup and point it to this machine's ip`,
-    `${T}3. A Clerk.com account`,
-    `${T}${T}- If you don't have one, you can create one at https://clerk.dev`,
-    `${T}${T}- We'll need NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, CLERK_SECRET_KEY, CLERK_SIGNING_SECRET`,
-    `${T}${T}- Create a webhook pointing to https://your_domain/api/webhook/clerk\n`,
-    'For more information you can read our article on self-hosting at https://docs.openpanel.dev/docs/self-hosting\n',
+    'For more information you can read our article on self-hosting at https://openpanel.dev/docs/self-hosting/self-hosting\n',
  ];

  console.log(
@@ -279,50 +268,6 @@ async function initiateOnboarding() {
    },
  ]);

-  // Clerk
-
-  const clerkResponse = await inquirer.prompt([
-    {
-      type: 'input',
-      name: 'NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY',
-      message: 'Enter your Clerk Publishable Key:',
-      default: process.env.DEBUG ? 'pk_test_1234567890' : undefined,
-      validate: (value) => {
-        if (value.startsWith('pk_live_') || value.startsWith('pk_test_')) {
-          return true;
-        }
-
-        return 'Please enter a valid Clerk Publishable Key. Should start with "pk_live_" or "pk_test_"';
-      },
-    },
-    {
-      type: 'input',
-      name: 'CLERK_SECRET_KEY',
-      message: 'Enter your Clerk Secret Key:',
-      default: process.env.DEBUG ? 'sk_test_1234567890' : undefined,
-      validate: (value) => {
-        if (value.startsWith('sk_live_') || value.startsWith('sk_test_')) {
-          return true;
-        }
-
-        return 'Please enter a valid Clerk Secret Key. Should start with "sk_live_" or "sk_test_"';
-      },
-    },
-    {
-      type: 'input',
-      name: 'CLERK_SIGNING_SECRET',
-      message: 'Enter your Clerk Signing Secret:',
-      default: process.env.DEBUG ? 'whsec_1234567890' : undefined,
-      validate: (value) => {
-        if (value.startsWith('whsec_')) {
-          return true;
-        }
-
-        return 'Please enter a valid Clerk Signing Secret. Should start with "whsec_"';
-      },
-    },
-  ]);
-
  // OS

  const cpus = await inquirer.prompt([
@@ -379,10 +324,7 @@ async function initiateOnboarding() {
      envs.DATABASE_URL ||
      'postgresql://postgres:postgres@op-db:5432/postgres?schema=public',
    DOMAIN_NAME: domainNameResponse.domainName,
-    NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY:
-      clerkResponse.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY || '',
-    CLERK_SECRET_KEY: clerkResponse.CLERK_SECRET_KEY || '',
-    CLERK_SIGNING_SECRET: clerkResponse.CLERK_SIGNING_SECRET || '',
+    COOKIE_SECRET: generatePassword(32),
  });

  console.log('Updating docker-compose.yml file...\n');
@@ -417,18 +359,16 @@ async function initiateOnboarding() {
      '======================================================================',
      'Here are some good things to know before you continue:',
      '',
-      `1. Make sure that your webhook is pointing at ${domainNameResponse.domainName}/api/webhook/clerk`,
-      '',
-      '2. Commands:',
+      '1. Commands:',
      '\t- ./start (example: ./start)',
      '\t- ./stop (example: ./stop)',
      '\t- ./logs (example: ./logs)',
      '\t- ./rebuild (example: ./rebuild op-dashboard)',
      '',
-      '3. Danger zone!',
+      '2. Danger zone!',
      '\t- ./danger_wipe_everything (example: ./danger_wipe_everything)',
      '',
-      '4. More about self-hosting: https://docs.openpanel.dev/docs/self-hosting',
+      '3. More about self-hosting: https://openpanel.dev/docs/self-hosting/self-hosting',
      '======================================================================',
      '',
      `Start OpenPanel with "./start" inside the self-hosting directory`,