Self-hosting! (#49)

* added self-hosting
2024-08-28 09:28:44 +02:00
parent f0b7526847
commit df05e2dab3
70 changed files with 2310 additions and 272 deletions
--- a/apps/worker/Dockerfile
+++ b/apps/worker/Dockerfile
@@ -1,85 +1,77 @@
 ARG NODE_VERSION=20.15.1

-FROM --platform=linux/amd64 node:${NODE_VERSION}-slim AS base
+FROM node:${NODE_VERSION}-slim AS base

 ARG DATABASE_URL
 ENV DATABASE_URL=$DATABASE_URL
-
 ENV PNPM_HOME="/pnpm"
-
 ENV PATH="$PNPM_HOME:$PATH"

-RUN corepack enable
-
-RUN apt update \
-    && apt install -y curl \
-    && curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n \
-    && bash n $NODE_VERSION \
-    && rm n \
-    && npm install -g n
+RUN corepack enable && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+    ca-certificates \
+    openssl \
+    libssl3 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*

 WORKDIR /app

-COPY package.json package.json
-COPY pnpm-lock.yaml pnpm-lock.yaml
-COPY pnpm-workspace.yaml pnpm-workspace.yaml
-COPY apps/worker/package.json apps/worker/package.json
-COPY packages/db/package.json packages/db/package.json
-COPY packages/redis/package.json packages/redis/package.json
-COPY packages/logger/package.json packages/logger/package.json
-COPY packages/queue/package.json packages/queue/package.json
-COPY packages/common/package.json packages/common/package.json
-COPY packages/constants/package.json packages/constants/package.json
-COPY packages/validation/package.json packages/validation/package.json
-COPY packages/sdks/sdk/package.json packages/sdks/sdk/package.json
-COPY patches patches
+# Workspace
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+# Apps
+COPY apps/worker/package.json ./apps/worker/
+# Packages
+COPY packages/db/package.json ./packages/db/
+COPY packages/redis/package.json ./packages/redis/
+COPY packages/queue/package.json ./packages/queue/
+COPY packages/logger/package.json ./packages/logger/
+COPY packages/common/package.json ./packages/common/
+COPY packages/constants/package.json ./packages/constants/
+# Patches
+COPY patches ./patches

 # BUILD
 FROM base AS build

-WORKDIR /app/apps/worker
-RUN pnpm install --frozen-lockfile --ignore-scripts
+WORKDIR /app
+RUN pnpm install --frozen-lockfile && \
+    pnpm store prune

-WORKDIR  /app
-COPY apps/worker apps/worker
-COPY packages packages
-COPY tooling tooling
-RUN pnpm db:codegen
+COPY apps/worker ./apps/worker
+COPY packages ./packages
+COPY tooling ./tooling

-WORKDIR /app/apps/worker
-RUN pnpm run build
+RUN pnpm db:codegen && \
+    pnpm --filter worker run build

 # PROD
 FROM base AS prod

-WORKDIR /app/apps/worker
-RUN pnpm install --frozen-lockfile --prod --ignore-scripts
+WORKDIR /app
+COPY --from=build /app/package.json ./
+COPY --from=build /app/pnpm-lock.yaml ./
+RUN pnpm install --frozen-lockfile --prod && \
+    pnpm store prune

 # FINAL
 FROM base AS runner

-COPY --from=build /app/package.json /app/package.json
-COPY --from=prod /app/node_modules /app/node_modules
+WORKDIR /app
+
+COPY --from=build /app/package.json ./
+COPY --from=prod /app/node_modules ./node_modules

 # Apps
-COPY --from=build /app/apps/worker /app/apps/worker
-
-# Apps node_modules
-COPY --from=prod /app/apps/worker/node_modules /app/apps/worker/node_modules
+COPY --from=build /app/apps/worker ./apps/worker

 # Packages
-COPY --from=build /app/packages/db /app/packages/db
-COPY --from=build /app/packages/redis /app/packages/redis
-COPY --from=build /app/packages/logger /app/packages/logger
-COPY --from=build /app/packages/queue /app/packages/queue
-COPY --from=build /app/packages/common /app/packages/common
-
-# Packages node_modules
-COPY --from=prod /app/packages/db/node_modules /app/packages/db/node_modules
-COPY --from=prod /app/packages/redis/node_modules /app/packages/redis/node_modules
-COPY --from=prod /app/packages/logger/node_modules /app/packages/logger/node_modules
-COPY --from=prod /app/packages/queue/node_modules /app/packages/queue/node_modules
-COPY --from=prod /app/packages/common/node_modules /app/packages/common/node_modules
+COPY --from=build /app/packages/db ./packages/db
+COPY --from=build /app/packages/redis ./packages/redis
+COPY --from=build /app/packages/logger ./packages/logger
+COPY --from=build /app/packages/queue ./packages/queue
+COPY --from=build /app/packages/common ./packages/common

 RUN pnpm db:codegen

--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -12,8 +12,8 @@
  },
  "dependencies": {
    "@baselime/pino-transport": "^0.1.5",
-    "@bull-board/api": "^5.21.0",
-    "@bull-board/express": "^5.21.0",
+    "@bull-board/api": "5.21.0",
+    "@bull-board/express": "5.21.0",
    "@openpanel/common": "workspace:*",
    "@openpanel/db": "workspace:*",
    "@openpanel/logger": "workspace:*",
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -5,6 +5,7 @@ import type { WorkerOptions } from 'bullmq';
 import { Worker } from 'bullmq';
 import express from 'express';

+import { createInitialSalts } from '@openpanel/db';
 import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';
 import { getRedisQueue } from '@openpanel/redis';

@@ -15,7 +16,7 @@ import { register } from './metrics';

 const PORT = parseInt(process.env.WORKER_PORT || '3000', 10);
 const serverAdapter = new ExpressAdapter();
-serverAdapter.setBasePath(process.env.SELF_HOSTED ? '/worker' : '/');
+serverAdapter.setBasePath('/');
 const app = express();

 const workerOptions: WorkerOptions = {
@@ -162,10 +163,28 @@ async function start() {
    }
  );

+  if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
+    await cronQueue.add(
+      'ping',
+      {
+        type: 'ping',
+        payload: undefined,
+      },
+      {
+        jobId: 'ping',
+        repeat: {
+          pattern: '0 0 * * *',
+        },
+      }
+    );
+  }
+
  const repeatableJobs = await cronQueue.getRepeatableJobs();

  console.log('Repeatable jobs:');
  console.log(repeatableJobs);
+
+  await createInitialSalts();
 }

 start();
--- a/apps/worker/src/jobs/cron.ping.ts
+++ b/apps/worker/src/jobs/cron.ping.ts
@@ -0,0 +1,26 @@
+import { chQuery, TABLE_NAMES } from '@openpanel/db';
+
+export async function ping() {
+  const [res] = await chQuery<{ count: number }>(
+    `SELECT COUNT(*) as count FROM ${TABLE_NAMES.events}`
+  );
+
+  if (typeof res?.count === 'number') {
+    const response = await fetch('https://api.openpanel.com/misc/ping', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        domain: process.env.NEXT_PUBLIC_DASHBOARD_URL,
+        count: res?.count,
+      }),
+    });
+
+    if (response.ok) {
+      return await response.json();
+    }
+
+    throw new Error('Failed to ping the server');
+  }
+}
--- a/apps/worker/src/jobs/cron.ts
+++ b/apps/worker/src/jobs/cron.ts
@@ -3,6 +3,7 @@ import type { Job } from 'bullmq';
 import { eventBuffer, profileBuffer } from '@openpanel/db';
 import type { CronQueuePayload } from '@openpanel/queue';

+import { ping } from './cron.ping';
 import { salt } from './cron.salt';

 export async function cronJob(job: Job<CronQueuePayload>) {
@@ -16,5 +17,8 @@ export async function cronJob(job: Job<CronQueuePayload>) {
    case 'flushProfiles': {
      return await profileBuffer.flush();
    }
+    case 'ping': {
+      return await ping();
+    }
  }
 }