improve: prepare for coolify and general self-hosting improvements (#175)

* fix(self-hosting): improve docker compose, add healthchecks, rename env SELF_HOSTED

* improve(db): improve initial migration when no data exists

* fix(db): misstakes were made

* improve(dashboard): better curl preview depending on project type

* fix(db): fix migrations

* fix(onboarding): ensure we publish event correctly

* wip

* fix: curl preview

* add coolify template

* fix(dashboard): page -> route

* fix

* fix env
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-06-23 22:21:11 +02:00
committed by GitHub
parent 4a2dbc5c4d
commit 92d62c3e5c
22 changed files with 382 additions and 60 deletions

View File

@@ -20,6 +20,7 @@ ENV PATH="$PNPM_HOME:$PATH"
RUN apt-get update && apt-get install -y \
openssl \
libssl3 \
curl \
&& rm -rf /var/lib/apt/lists/*
RUN corepack enable
@@ -62,6 +63,7 @@ WORKDIR /app/apps/dashboard
# Will be replaced on runtime
ENV NEXT_PUBLIC_DASHBOARD_URL="__NEXT_PUBLIC_DASHBOARD_URL__"
ENV NEXT_PUBLIC_API_URL="__NEXT_PUBLIC_API_URL__"
ENV NEXT_PUBLIC_SELF_HOSTED="__NEXT_PUBLIC_SELF_HOSTED__"
RUN pnpm run build

View File

@@ -4,7 +4,7 @@ set -e
echo "> Replace env variable placeholders with runtime values..."
# Define environment variables to check (space-separated string)
variables_to_replace="NEXT_PUBLIC_DASHBOARD_URL NEXT_PUBLIC_API_URL"
variables_to_replace="NEXT_PUBLIC_DASHBOARD_URL NEXT_PUBLIC_API_URL NEXT_PUBLIC_SELF_HOSTED"
# Replace env variable placeholders with real values
for key in $variables_to_replace; do

View File

@@ -99,7 +99,7 @@ export default function LayoutMenu({
</div>
</ProjectLink>
)}
{process.env.SELF_HOSTED && (
{process.env.NEXT_PUBLIC_SELF_HOSTED === 'true' && (
<a
className="rounded p-2 row items-center gap-2 hover:bg-def-200"
href="https://openpanel.dev/supporter"
@@ -231,7 +231,7 @@ export default function LayoutMenu({
))}
</div>
</div>
{process.env.SELF_HOSTED && (
{process.env.NEXT_PUBLIC_SELF_HOSTED === 'true' && (
<div className="mt-auto w-full ">
<div className={cn('text-sm w-full text-center')}>
Self-hosted instance

View File

@@ -27,7 +27,7 @@ export default async function Page({
params: { organizationSlug: organizationId },
searchParams,
}: PageProps) {
const isBillingEnabled = !process.env.SELF_HOSTED;
const isBillingEnabled = process.env.NEXT_PUBLIC_SELF_HOSTED !== 'true';
const tab = parseAsStringEnum(['org', 'billing', 'members', 'invites'])
.withDefault('org')
.parseServerSide(searchParams.tab);

View File

@@ -104,22 +104,34 @@ function CurlPreview({ project }: { project: IServiceProjectWithClients }) {
return null;
}
const payload: Record<string, any> = {
type: 'track',
payload: {
name: 'screen_view',
properties: {
__title: `Testing OpenPanel - ${project.name}`,
__path: `${project.domain}`,
__referrer: `${process.env.NEXT_PUBLIC_DASHBOARD_URL}`,
},
},
};
if (project.types.includes('app')) {
payload.payload.properties.__path = '/';
delete payload.payload.properties.__referrer;
}
if (project.types.includes('backend')) {
payload.payload.name = 'test_event';
payload.payload.properties = {};
}
const code = `curl -X POST ${process.env.NEXT_PUBLIC_API_URL}/track \\
-H "Content-Type: application/json" \\
-H "openpanel-client-id: ${client.id}" \\
-H "openpanel-client-secret: ${secret}" \\
-H "User-Agent: ${window.navigator.userAgent}" \\
-d '{
"type": "track",
"payload": {
"name": "screen_view",
"properties": {
"__title": "Testing OpenPanel - ${project.name}",
"__path": "${project.domain}",
"__referrer": "${process.env.NEXT_PUBLIC_DASHBOARD_URL}"
}
}
}'`;
-H "User-Agent: ${typeof window !== 'undefined' ? window.navigator.userAgent : ''}" \\
-d '${JSON.stringify(payload)}'`;
return (
<div className="card">

View File

@@ -27,7 +27,7 @@ const Verify = async ({ params: { projectId } }: Props) => {
const [project, events] = await Promise.all([
await getProjectWithClients(projectId),
getEvents(
`SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} LIMIT 100`,
`SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 100`,
),
]);
@@ -35,7 +35,7 @@ const Verify = async ({ params: { projectId } }: Props) => {
return <div>Hmm, something fishy is going on. Please reload the page.</div>;
}
return <OnboardingVerify project={project} events={events} />;
return <OnboardingVerify project={project} events={events.reverse()} />;
};
export default Verify;

View File

@@ -1,7 +0,0 @@
export const runtime = 'edge';
export const dynamic = 'force-dynamic'; // no caching
export async function GET(request: Request) {
const headers = Object.fromEntries(request.headers.entries());
return Response.json({ headers, region: process.env.VERCEL_REGION });
}

View File

@@ -0,0 +1,5 @@
export const dynamic = 'force-dynamic'; // no caching
export async function GET(request: Request) {
return Response.json({ status: 'ok' });
}

View File

@@ -3,6 +3,14 @@ title: Changelog for self-hosting
description: This is a list of changes that have been made to the self-hosting setup.
---
## 1.2.0
We have renamed `SELF_HOSTED` to `NEXT_PUBLIC_SELF_HOSTED`. It's important to rename this env before your upgrade to this version.
## 1.1.1
Packed with new features since our first stable release.
## 1.0.0 (stable)
OpenPanel self-hosting is now in a stable state and should not be any breaking changes in the future.

View File

@@ -15,6 +15,7 @@ RUN corepack enable && \
apt-get install -y --no-install-recommends \
ca-certificates \
openssl \
curl \
libssl3 && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

View File

@@ -36,7 +36,11 @@ export async function bootCron() {
},
];
if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {
if (
(process.env.NEXT_PUBLIC_SELF_HOSTED === 'true' ||
process.env.SELF_HOSTED) &&
process.env.NODE_ENV === 'production'
) {
jobs.push({
name: 'ping',
type: 'ping',

View File

@@ -57,6 +57,10 @@ async function start() {
});
});
app.get('/healthcheck', (req, res) => {
res.json({ status: 'ok' });
});
app.listen(PORT, () => {
console.log(`For the UI, open http://localhost:${PORT}/`);
});

View File

@@ -46,9 +46,10 @@ export async function deleteProjects(job: Job<CronQueuePayload>) {
];
for (const table of tables) {
const query = process.env.SELF_HOSTED
? `ALTER TABLE ${table} DELETE WHERE ${where};`
: `ALTER TABLE ${table}_replicated ON CLUSTER '{cluster}' DELETE WHERE ${where};`;
const query =
process.env.NEXT_PUBLIC_SELF_HOSTED === 'true'
? `ALTER TABLE ${table} DELETE WHERE ${where};`
: `ALTER TABLE ${table}_replicated ON CLUSTER '{cluster}' DELETE WHERE ${where};`;
await ch.command({
query,

View File

@@ -11,7 +11,7 @@ import {
renameTable,
runClickhouseMigrationCommands,
} from '../src/clickhouse/migration';
import { printBoxMessage } from './helpers';
import { getIsSelfHosting, printBoxMessage } from './helpers';
export async function up() {
const replicatedVersion = '1';
@@ -25,7 +25,7 @@ export async function up() {
'profile_aliases_distributed',
);
const isSelfHosting = !!process.env.SELF_HOSTED;
const isSelfHosting = getIsSelfHosting();
const isClustered = !isSelfHosting;
const isSelfHostingPostCluster =

View File

@@ -1,7 +1,8 @@
import fs from 'node:fs';
import path from 'node:path';
import { formatClickhouseDate } from '../src/clickhouse/client';
import { TABLE_NAMES, formatClickhouseDate } from '../src/clickhouse/client';
import {
chMigrationClient,
createTable,
runClickhouseMigrationCommands,
} from '../src/clickhouse/migration';
@@ -66,7 +67,7 @@ export async function up() {
}),
];
sqls.push(...createOldSessions());
sqls.push(...(await createOldSessions()));
fs.writeFileSync(
path.join(__filename.replace('.ts', '.sql')),
@@ -86,8 +87,31 @@ export async function up() {
}
}
function createOldSessions() {
let startDate = new Date('2024-03-01');
async function createOldSessions() {
async function getFirstEventAt() {
const defaultDate = new Date('2024-03-01');
try {
const res = await chMigrationClient.query({
query: `SELECT min(created_at) as created_at, count() as count FROM ${TABLE_NAMES.events}`,
format: 'JSONEachRow',
});
const json = await res.json<{ created_at: string; count: string }>();
const row = json[0];
if (!row || row.count === '0') {
return null;
}
return new Date(row.created_at);
} catch (e) {
return defaultDate;
}
}
let startDate = await getFirstEventAt();
if (!startDate) {
return [];
}
const endDate = new Date();
const sqls: string[] = [];
while (startDate <= endDate) {

View File

@@ -24,7 +24,9 @@ export function getIsCluster() {
}
export function getIsSelfHosting() {
return !!process.env.SELF_HOSTED;
return (
process.env.NEXT_PUBLIC_SELF_HOSTED === 'true' || !!process.env.SELF_HOSTED
);
}
export function getIsDry() {

View File

@@ -261,7 +261,7 @@ return "OK"
if (!_multi) {
await multi.exec();
}
await publishEvent('events', 'received', transformEvent(event), multi);
await publishEvent('events', 'received', transformEvent(event));
} catch (error) {
this.logger.error('Failed to add event to Redis buffer', { error });
}

View File

@@ -59,12 +59,20 @@ const getPrismaClient = () => {
subscriptionStatus: {
needs: { subscriptionStatus: true, subscriptionCanceledAt: true },
compute(org) {
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return 'active';
}
return org.subscriptionStatus || 'trialing';
},
},
hasSubscription: {
needs: { subscriptionStatus: true, subscriptionEndsAt: true },
compute(org) {
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return false;
}
if (
[null, 'canceled', 'trialing'].includes(org.subscriptionStatus)
) {
@@ -86,6 +94,10 @@ const getPrismaClient = () => {
subscriptionPeriodEventsCountExceededAt: true,
},
compute(org) {
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return null;
}
if (
org.subscriptionEndsAt &&
org.subscriptionPeriodEventsCountExceededAt
@@ -119,6 +131,10 @@ const getPrismaClient = () => {
isCanceled: {
needs: { subscriptionStatus: true, subscriptionCanceledAt: true },
compute(org) {
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return false;
}
return isCanceled(org);
},
},
@@ -129,6 +145,10 @@ const getPrismaClient = () => {
subscriptionEndsAt: true,
},
compute(org) {
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return false;
}
return isWillBeCanceled(org);
},
},
@@ -139,6 +159,10 @@ const getPrismaClient = () => {
subscriptionCanceledAt: true,
},
compute(org) {
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return false;
}
if (isCanceled(org)) {
return false;
}
@@ -158,6 +182,10 @@ const getPrismaClient = () => {
subscriptionPeriodEventsLimit: true,
},
compute(org) {
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return false;
}
return (
org.subscriptionPeriodEventsCount >
org.subscriptionPeriodEventsLimit
@@ -167,7 +195,13 @@ const getPrismaClient = () => {
subscriptionCurrentPeriodStart: {
needs: { subscriptionStartsAt: true, subscriptionInterval: true },
compute(org) {
if (!org.subscriptionStartsAt) return org.subscriptionStartsAt;
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return null;
}
if (!org.subscriptionStartsAt) {
return null;
}
if (org.subscriptionInterval === 'year') {
const startDay = org.subscriptionStartsAt.getUTCDate();
@@ -195,7 +229,13 @@ const getPrismaClient = () => {
subscriptionInterval: true,
},
compute(org) {
if (!org.subscriptionStartsAt) return org.subscriptionEndsAt;
if (process.env.NEXT_PUBLIC_SELF_HOSTED === 'true') {
return null;
}
if (!org.subscriptionStartsAt) {
return null;
}
if (org.subscriptionInterval === 'year') {
const startDay = org.subscriptionStartsAt.getUTCDate();

View File

@@ -33,7 +33,10 @@ async function createOrGetOrganization(
},
});
if (!process.env.SELF_HOSTED) {
if (
process.env.NEXT_PUBLIC_SELF_HOSTED !== 'true' &&
!process.env.SELF_HOSTED
) {
await addTrialEndingSoonJob(
organization.id,
1000 * 60 * 60 * 24 * TRIAL_DURATION_IN_DAYS * 0.9,

View File

@@ -1,5 +1,5 @@
NODE_ENV="production"
SELF_HOSTED="true"
NEXT_PUBLIC_SELF_HOSTED="true"
GEO_IP_HOST="http://op-geo:8080"
BATCH_SIZE="5000"
BATCH_INTERVAL="10000"

208
self-hosting/coolify.yml Normal file
View File

@@ -0,0 +1,208 @@
# documentation: https://openpanel.dev/docs
# slogan: Open source alternative to Mixpanel and Plausible for product analytics
# tags: analytics, insights, privacy, mixpanel, plausible, google, alternative
# logo: svgs/openpanel.svg
# port: 3000
services:
opdb:
image: postgres:16-alpine
restart: always
volumes:
- opdb-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=${OPENPANEL_POSTGRES_DB:-openpanel-db}
- POSTGRES_USER=${SERVICE_USER_POSTGRES}
- POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES}
healthcheck:
test: [CMD-SHELL, "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 10s
timeout: 5s
retries: 5
opkv:
image: redis:7.4-alpine
restart: always
volumes:
- opkv-data:/data
command: redis-server --requirepass ${SERVICE_PASSWORD_REDIS} --maxmemory-policy noeviction
healthcheck:
test: [CMD, redis-cli, -a, "${SERVICE_PASSWORD_REDIS}", ping]
interval: 10s
timeout: 5s
retries: 5
opch:
image: clickhouse/clickhouse-server:24.3.2-alpine
restart: always
volumes:
- opch-data:/var/lib/clickhouse
- opch-logs:/var/log/clickhouse-server
- type: bind
source: ./clickhouse-config.xml
target: /etc/clickhouse-server/config.d/op-config.xml
read_only: true
content: |
<clickhouse>
<logger>
<level>warning</level>
<console>true</console>
</logger>
<keep_alive_timeout>10</keep_alive_timeout>
<!-- Stop all the unnecessary logging -->
<query_thread_log remove="remove"/>
<query_log remove="remove"/>
<text_log remove="remove"/>
<trace_log remove="remove"/>
<metric_log remove="remove"/>
<asynchronous_metric_log remove="remove"/>
<session_log remove="remove"/>
<part_log remove="remove"/>
<listen_host>0.0.0.0</listen_host>
<interserver_listen_host>0.0.0.0</interserver_listen_host>
<interserver_http_host>opch</interserver_http_host>
<!-- Disable cgroup memory observer -->
<cgroups_memory_usage_observer_wait_time>0</cgroups_memory_usage_observer_wait_time>
<!-- Not used anymore, but kept for backwards compatibility -->
<macros>
<shard>1</shard>
<replica>replica1</replica>
<cluster>openpanel_cluster</cluster>
</macros>
</clickhouse>
- type: bind
source: ./clickhouse-user-config.xml
target: /etc/clickhouse-server/users.d/op-user-config.xml
read_only: true
content: |
<clickhouse>
<profiles>
<default>
<log_queries>0</log_queries>
<log_query_threads>0</log_query_threads>
</default>
</profiles>
</clickhouse>
- type: bind
source: ./init-db.sh
target: /docker-entrypoint-initdb.d/init-db.sh
content: |
#!/bin/sh
set -e
clickhouse client -n <<-EOSQL
CREATE DATABASE IF NOT EXISTS openpanel;
EOSQL
healthcheck:
test: [CMD-SHELL, 'clickhouse-client --query "SELECT 1"']
interval: 10s
timeout: 5s
retries: 5
ulimits:
nofile:
soft: 262144
hard: 262144
opapi:
image: lindesvard/openpanel-api:1.0.0
restart: always
command: >
sh -c "
echo 'Waiting for PostgreSQL to be ready...'
while ! nc -z opdb 5432; do
sleep 1
done
echo 'PostgreSQL is ready'
echo 'Waiting for ClickHouse to be ready...'
while ! nc -z opch 8123; do
sleep 1
done
echo 'ClickHouse is ready'
echo 'Running migrations...'
CI=true pnpm -r run migrate:deploy
pnpm start
"
depends_on:
opdb:
condition: service_healthy
opch:
condition: service_healthy
opkv:
condition: service_healthy
environment:
# Common
- NODE_ENV=production
- NEXT_PUBLIC_SELF_HOSTED=true
# URLs
- DATABASE_URL=postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@opdb:5432/${OPENPANEL_POSTGRES_DB:-openpanel-db}?schema=public
- DATABASE_URL_DIRECT=postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@opdb:5432/${OPENPANEL_POSTGRES_DB:-openpanel-db}?schema=public
- REDIS_URL=redis://default:${SERVICE_PASSWORD_REDIS}@opkv:6379
- CLICKHOUSE_URL=${OPENPANEL_CLICKHOUSE_URL:-http://opch:8123/openpanel}
- SERVICE_FQDN_OPAPI=/api
# Set coolify FQDN domain
- NEXT_PUBLIC_API_URL=$SERVICE_FQDN_OPAPI
- NEXT_PUBLIC_DASHBOARD_URL=$SERVICE_FQDN_OPDASHBOARD
# Others
- COOKIE_SECRET=${SERVICE_BASE64_COOKIESECRET}
- ALLOW_REGISTRATION=${OPENPANEL_ALLOW_REGISTRATION:-false}
- ALLOW_INVITATION=${OPENPANEL_ALLOW_INVITATION:-true}
- EMAIL_SENDER=${OPENPANEL_EMAIL_SENDER}
- RESEND_API_KEY=${RESEND_API_KEY}
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
opdashboard:
image: lindesvard/openpanel-dashboard:1.0.0
restart: always
depends_on:
opapi:
condition: service_healthy
environment:
# Common
- NODE_ENV=production
- NEXT_PUBLIC_SELF_HOSTED=true
# URLs
- DATABASE_URL=postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@opdb:5432/${OPENPANEL_POSTGRES_DB:-openpanel-db}?schema=public
- REDIS_URL=redis://default:${SERVICE_PASSWORD_REDIS}@opkv:6379
- CLICKHOUSE_URL=${OPENPANEL_CLICKHOUSE_URL:-http://opch:8123/openpanel}
- SERVICE_FQDN_OPDASHBOARD
# Set coolify FQDN domain
- NEXT_PUBLIC_API_URL=$SERVICE_FQDN_OPAPI
- NEXT_PUBLIC_DASHBOARD_URL=$SERVICE_FQDN_OPDASHBOARD
healthcheck:
test:
["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
opworker:
image: lindesvard/openpanel-worker:1.0.0
restart: always
depends_on:
opapi:
condition: service_healthy
environment:
# FQDN
- SERVICE_FQDN_OPBULLBOARD
# Common
- NODE_ENV=production
- NEXT_PUBLIC_SELF_HOSTED=true
# URLs
- DATABASE_URL=postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@opdb:5432/${OPENPANEL_POSTGRES_DB:-openpanel-db}?schema=public
- DATABASE_URL_DIRECT=postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@opdb:5432/${OPENPANEL_POSTGRES_DB:-openpanel-db}?schema=public
- REDIS_URL=redis://default:${SERVICE_PASSWORD_REDIS}@opkv:6379
- CLICKHOUSE_URL=${OPENPANEL_CLICKHOUSE_URL:-http://opch:8123/openpanel}
# Set coolify FQDN domain
- NEXT_PUBLIC_API_URL=$SERVICE_FQDN_OPAPI
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5

View File

@@ -12,8 +12,10 @@ services:
- op-proxy-config:/config
- ./caddy/Caddyfile:/etc/caddy/Caddyfile
depends_on:
- op-dashboard
- op-api
op-dashboard:
condition: service_healthy
op-api:
condition: service_healthy
op-db:
image: postgres:14-alpine
@@ -38,6 +40,11 @@ services:
volumes:
- op-kv-data:/data
command: [ 'redis-server', '--maxmemory-policy', 'noeviction' ]
healthcheck:
test: [ 'CMD-SHELL', 'redis-cli ping' ]
interval: 10s
timeout: 5s
retries: 5
# Uncomment to expose ports
# ports:
# - 6379:6379
@@ -66,27 +73,23 @@ services:
restart: always
command: >
sh -c "
echo 'Waiting for PostgreSQL to be ready...'
while ! nc -z op-db 5432; do
sleep 1
done
echo 'PostgreSQL is ready'
echo 'Waiting for ClickHouse to be ready...'
while ! nc -z op-ch 8123; do
sleep 1
done
echo 'ClickHouse is ready'
echo 'Running migrations...'
CI=true pnpm -r run migrate:deploy
pnpm start
"
healthcheck:
test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/healthcheck || exit 1' ]
interval: 10s
timeout: 5s
retries: 5
depends_on:
- op-db
- op-ch
- op-kv
op-db:
condition: service_healthy
op-ch:
condition: service_healthy
op-kv:
condition: service_healthy
env_file:
- .env
@@ -94,17 +97,29 @@ services:
image: lindesvard/openpanel-dashboard:latest
restart: always
depends_on:
- op-api
op-api:
condition: service_healthy
env_file:
- .env
healthcheck:
test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/api/healthcheck || exit 1' ]
interval: 10s
timeout: 5s
retries: 5
op-worker:
image: lindesvard/openpanel-worker:latest
restart: always
depends_on:
- op-api
op-api:
condition: service_healthy
env_file:
- .env
healthcheck:
test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/healthcheck || exit 1' ]
interval: 10s
timeout: 5s
retries: 5
deploy:
mode: replicated
replicas: $OP_WORKER_REPLICAS