diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts index b5d00528..2ee11fc9 100644 --- a/apps/api/src/controllers/track.controller.ts +++ b/apps/api/src/controllers/track.controller.ts @@ -1,6 +1,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify'; import { assocPath, pathOr, pick } from 'ramda'; +import { HttpError } from '@/utils/errors'; import { generateId, slug } from '@openpanel/common'; import { generateDeviceId, parseUserAgent } from '@openpanel/common/server'; import { getProfileById, getSalts, upsertProfile } from '@openpanel/db'; @@ -187,9 +188,16 @@ export async function handler( break; } case 'identify': { + const payload = request.body.payload; const geo = await getGeoLocation(ip); + if (!payload.profileId) { + throw new HttpError('Missing profileId', { + status: 400, + }); + } + await identify({ - payload: request.body.payload, + payload, projectId, geo, ua, diff --git a/apps/worker/src/jobs/cron.ping.ts b/apps/worker/src/jobs/cron.ping.ts index 8252702b..547ce83a 100644 --- a/apps/worker/src/jobs/cron.ping.ts +++ b/apps/worker/src/jobs/cron.ping.ts @@ -1,6 +1,10 @@ import { TABLE_NAMES, chQuery } from '@openpanel/db'; export async function ping() { + if (process.env.DISABLE_PING) { + return; + } + const [res] = await chQuery<{ count: number }>( `SELECT COUNT(*) as count FROM ${TABLE_NAMES.events}`, ); diff --git a/packages/db/code-migrations/8-order-keys.ts b/packages/db/code-migrations/8-order-keys.ts index 244b19d2..82686641 100644 --- a/packages/db/code-migrations/8-order-keys.ts +++ b/packages/db/code-migrations/8-order-keys.ts @@ -9,25 +9,6 @@ import { } from '../src/clickhouse/migration'; import { getIsCluster } from './helpers'; -/** - * Migration to update ORDER BY keys for events and sessions tables. - * - * Changes: - * - Events: Remove profile_id from ORDER BY, add created_at for better ordering - * Old: ['project_id', 'toDate(created_at)', 'profile_id', 'name'] - * New: ['project_id', 'toDate(created_at)', 'created_at', 'name'] - * - * - Sessions: Remove profile_id from ORDER BY, reorder to match query patterns - * Old: ['project_id', 'id', 'toDate(created_at)', 'profile_id'] - * New: ['project_id', 'toDate(created_at)', 'created_at', 'id'] - * - * Rationale: - * - project_id: Always filtered first (100% of queries) - * - toDate(created_at): Almost always filtered (95%+ of queries), good for partitioning - * - created_at: Helps with ordering within same day, matches ORDER BY patterns in queries - * - name (events): Frequently filtered (screen_view, session_start, etc.), good selectivity - * - id (sessions): Used for ordering and uniqueness in session queries - */ export async function up() { const isClustered = getIsCluster(); diff --git a/packages/db/code-migrations/helpers.ts b/packages/db/code-migrations/helpers.ts index 128a3ead..38e060d7 100644 --- a/packages/db/code-migrations/helpers.ts +++ b/packages/db/code-migrations/helpers.ts @@ -30,3 +30,7 @@ export function getIsSelfHosting() { export function getIsDry() { return process.argv.includes('--dry'); } + +export function getShouldIgnoreRecord() { + return process.argv.includes('--no-record'); +} diff --git a/packages/db/code-migrations/migrate.ts b/packages/db/code-migrations/migrate.ts index fa1ff99f..1b10b37d 100644 --- a/packages/db/code-migrations/migrate.ts +++ b/packages/db/code-migrations/migrate.ts @@ -10,6 +10,7 @@ import { getIsCluster, getIsDry, getIsSelfHosting, + getShouldIgnoreRecord, printBoxMessage, } from './helpers'; @@ -55,8 +56,8 @@ async function migrate() { ]); if (!getIsSelfHosting()) { - printBoxMessage('🕒 Migrations starts in 10 seconds', []); if (!getIsDry()) { + printBoxMessage('🕒 Migrations starts in 10 seconds', []); await new Promise((resolve) => setTimeout(resolve, 10000)); } } @@ -83,7 +84,7 @@ async function runMigration(migrationsDir: string, file: string) { try { const migration = await import(path.join(migrationsDir, file)); await migration.up(); - if (!getIsDry()) { + if (!getIsDry() && !getShouldIgnoreRecord()) { await db.codeMigration.upsert({ where: { name: file, diff --git a/packages/db/src/clickhouse/migration.ts b/packages/db/src/clickhouse/migration.ts index f4fcbf41..c94512f9 100644 --- a/packages/db/src/clickhouse/migration.ts +++ b/packages/db/src/clickhouse/migration.ts @@ -318,18 +318,13 @@ export function moveDataBetweenTables({ break; } - // For monthly/weekly intervals with transform, we need to use the start of the next period for the upper bound + // For monthly/weekly intervals with transform, upperBoundDate should be currentDate + // because currentDate already represents the start of the period we're processing + // The WHERE clause uses > previousDate AND <= currentDate to get exactly one period let upperBoundDate = currentDate; - if (interval === 'month' && batch.transform) { - const nextMonth = new Date(currentDate); - nextMonth.setMonth(nextMonth.getMonth() + 1); - nextMonth.setDate(1); - upperBoundDate = nextMonth; - } else if (interval === 'week' && batch.transform) { - const nextWeek = new Date(currentDate); - nextWeek.setDate(nextWeek.getDate() + 7); - const nextWeekStart = getWeekStart(nextWeek); - upperBoundDate = nextWeekStart; + // Don't exceed the endDate + if (upperBoundDate > endDate) { + upperBoundDate = endDate; } const sql = `INSERT INTO ${to} diff --git a/self-hosting/docker-compose.template.yml b/self-hosting/docker-compose.template.yml index 64c16c2a..205d5dc4 100644 --- a/self-hosting/docker-compose.template.yml +++ b/self-hosting/docker-compose.template.yml @@ -1,12 +1,12 @@ -version: '3' +version: "3" services: op-proxy: image: caddy:2-alpine restart: always ports: - - '80:80' - - '443:443' + - "80:80" + - "443:443" volumes: - op-proxy-data:/data - op-proxy-config:/config @@ -28,7 +28,7 @@ services: volumes: - op-db-data:/var/lib/postgresql/data healthcheck: - test: [ 'CMD-SHELL', 'pg_isready -U postgres' ] + test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s timeout: 5s retries: 5 @@ -49,9 +49,9 @@ services: restart: always volumes: - op-kv-data:/data - command: [ 'redis-server', '--maxmemory-policy', 'noeviction' ] + command: ["redis-server", "--maxmemory-policy", "noeviction"] healthcheck: - test: [ 'CMD-SHELL', 'redis-cli ping' ] + test: ["CMD-SHELL", "redis-cli ping"] interval: 10s timeout: 5s retries: 5 @@ -65,7 +65,7 @@ services: # - 6379:6379 op-ch: - image: clickhouse/clickhouse-server:24.3.2-alpine + image: clickhouse/clickhouse-server:25.10.2.65 restart: always volumes: - op-ch-data:/var/lib/clickhouse @@ -74,7 +74,7 @@ services: - ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro - ./clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro healthcheck: - test: [ 'CMD-SHELL', 'clickhouse-client --query "SELECT 1"' ] + test: ["CMD-SHELL", 'clickhouse-client --query "SELECT 1"'] interval: 10s timeout: 5s retries: 5 @@ -89,7 +89,7 @@ services: max-file: "3" op-api: - image: lindesvard/openpanel-api:latest + image: lindesvard/openpanel-api:2.0.0 restart: always command: > sh -c " @@ -99,7 +99,7 @@ services: pnpm start " healthcheck: - test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/healthcheck || exit 1' ] + test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -119,7 +119,7 @@ services: max-file: "3" op-dashboard: - image: lindesvard/openpanel-dashboard:latest + image: lindesvard/openpanel-dashboard:2.0.0 restart: always depends_on: op-api: @@ -127,7 +127,8 @@ services: env_file: - .env healthcheck: - test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/api/healthcheck || exit 1' ] + test: + ["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -138,7 +139,7 @@ services: max-file: "3" op-worker: - image: lindesvard/openpanel-worker:latest + image: lindesvard/openpanel-worker:2.0.0 restart: always depends_on: op-api: @@ -146,7 +147,7 @@ services: env_file: - .env healthcheck: - test: [ 'CMD-SHELL', 'curl -f http://localhost:3000/healthcheck || exit 1' ] + test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"] interval: 10s timeout: 5s retries: 5 @@ -171,4 +172,4 @@ volumes: op-proxy-data: driver: local op-proxy-config: - driver: local \ No newline at end of file + driver: local