diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts index bcce6a80..55652de8 100644 --- a/apps/worker/src/boot-workers.ts +++ b/apps/worker/src/boot-workers.ts @@ -117,7 +117,7 @@ export async function bootWorkers() { const worker = new GroupWorker({ queue, concurrency, - logger: queueLogger, + logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined, blockingTimeoutSec: Number.parseFloat( process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1', ), diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.ts index 83fac06a..d24e554e 100644 --- a/apps/worker/src/jobs/events.create-session-end.ts +++ b/apps/worker/src/jobs/events.create-session-end.ts @@ -96,7 +96,6 @@ export async function createSessionEnd( ...payload, properties: { ...payload.properties, - ...(session?.properties ?? {}), __bounce: session.is_bounce, }, name: 'session_end', diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.ts index 9b3a54c7..1ee1ebb9 100644 --- a/apps/worker/src/jobs/events.incoming-events.test.ts +++ b/apps/worker/src/jobs/events.incoming-events.test.ts @@ -319,7 +319,6 @@ describe('incomingEvent', () => { utm_content: '', utm_medium: '', revenue: 0, - properties: {}, project_id: projectId, device_id: 'last-device-123', profile_id: 'profile-123', diff --git a/packages/db/code-migrations/8-order-keys.ts b/packages/db/code-migrations/8-order-keys.ts index 82686641..4fe699dd 100644 --- a/packages/db/code-migrations/8-order-keys.ts +++ b/packages/db/code-migrations/8-order-keys.ts @@ -115,7 +115,6 @@ export async function up() { '`referrer_type` LowCardinality(String)', '`sign` Int8', '`version` UInt64', - '`properties` Map(String, String) CODEC(ZSTD(3))', ], // New ORDER BY: project_id, toDate(created_at), created_at, id // Removed profile_id, reordered to match query patterns (date first, then id) @@ -175,6 +174,45 @@ export async function up() { ...moveDataBetweenTables({ from: 'sessions', to: 'sessions_new_20251123', + columns: [ + 'id', + 'project_id', + 'profile_id', + 'device_id', + 'created_at', + 'ended_at', + 'is_bounce', + 'entry_origin', + 'entry_path', + 'exit_origin', + 'exit_path', + 'screen_view_count', + 'revenue', + 'event_count', + 'duration', + 'country', + 'region', + 'city', + 'longitude', + 'latitude', + 'device', + 'brand', + 'model', + 'browser', + 'browser_version', + 'os', + 'os_version', + 'utm_medium', + 'utm_source', + 'utm_campaign', + 'utm_content', + 'utm_term', + 'referrer', + 'referrer_name', + 'referrer_type', + 'sign', + 'version', + ], batch: { startDate: firstSessionDate, column: 'toDate(created_at)', diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.ts index 4d82eb53..fad5ee09 100644 --- a/packages/db/src/buffers/session-buffer.ts +++ b/packages/db/src/buffers/session-buffer.ts @@ -1,6 +1,5 @@ import { type Redis, getRedisCache } from '@openpanel/redis'; -import { toDots } from '@openpanel/common'; import { getSafeJson } from '@openpanel/json'; import { assocPath, clone } from 'ramda'; import { TABLE_NAMES, ch } from '../clickhouse/client'; @@ -91,10 +90,6 @@ export class SessionBuffer extends BaseBuffer { session: newSession, }); } - newSession.properties = toDots({ - ...(event.properties || {}), - ...(newSession.properties || {}), - }); const addedRevenue = event.name === 'revenue' ? (event.revenue ?? 0) : 0; newSession.revenue = (newSession.revenue ?? 0) + addedRevenue; @@ -168,7 +163,6 @@ export class SessionBuffer extends BaseBuffer { : '', sign: 1, version: 1, - properties: toDots(event.properties || {}), }, ]; } diff --git a/packages/db/src/clickhouse/migration.ts b/packages/db/src/clickhouse/migration.ts index c94512f9..2b2241d4 100644 --- a/packages/db/src/clickhouse/migration.ts +++ b/packages/db/src/clickhouse/migration.ts @@ -217,6 +217,7 @@ export function moveDataBetweenTables({ from, to, batch, + columns, }: { from: string; to: string; @@ -227,11 +228,15 @@ export function moveDataBetweenTables({ endDate?: Date; startDate?: Date; }; + columns?: string[]; }): string[] { const sqls: string[] = []; + // Build the SELECT clause + const selectClause = columns && columns.length > 0 ? columns.join(', ') : '*'; + if (!batch) { - return [`INSERT INTO ${to} SELECT * FROM ${from}`]; + return [`INSERT INTO ${to} SELECT ${selectClause} FROM ${from}`]; } // Start from today and go back 3 years @@ -328,7 +333,7 @@ export function moveDataBetweenTables({ } const sql = `INSERT INTO ${to} - SELECT * FROM ${from} + SELECT ${selectClause} FROM ${from} WHERE ${batch.column} > '${batch.transform ? batch.transform(previousDate) : formatClickhouseDate(previousDate, true)}' AND ${batch.column} <= '${batch.transform ? batch.transform(upperBoundDate) : formatClickhouseDate(upperBoundDate, true)}'`; sqls.push(sql); diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index e79ed8bb..e0b5322e 100644 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -131,7 +131,6 @@ export function transformSessionToEvent( duration: 0, revenue: session.revenue, properties: { - ...session.properties, is_bounce: session.is_bounce, __query: { utm_medium: session.utm_medium, diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.ts index 0d3bcccc..0f8eb6f4 100644 --- a/packages/db/src/services/session.service.ts +++ b/packages/db/src/services/session.service.ts @@ -1,6 +1,5 @@ import { cacheable } from '@openpanel/redis'; import type { IChartEventFilter } from '@openpanel/validation'; -import { uniq } from 'ramda'; import sqlstring from 'sqlstring'; import { TABLE_NAMES, @@ -53,7 +52,6 @@ export type IClickhouseSession = { revenue: number; sign: 1 | 0; version: number; - properties: Record; }; export interface IServiceSession { @@ -92,7 +90,6 @@ export interface IServiceSession { utmContent: string; utmTerm: string; revenue: number; - properties: Record; profile?: IServiceProfile; } @@ -144,7 +141,6 @@ export function transformSession(session: IClickhouseSession): IServiceSession { utmContent: session.utm_content, utmTerm: session.utm_term, revenue: session.revenue, - properties: session.properties, profile: undefined, }; } diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts index e1e64645..d21bf4e7 100644 --- a/packages/queue/src/queues.ts +++ b/packages/queue/src/queues.ts @@ -146,7 +146,7 @@ export const eventsGroupQueues = Array.from({ }).map( (_, index, list) => new GroupQueue({ - logger: queueLogger, + logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined, namespace: getQueueName( list.length === 1 ? 'group_events' : `group_events_${index}`, ),