feat:add otel logging

apps/worker/package.json

@@ -24,6 +24,7 @@
     "@openpanel/payments": "workspace:*",
     "@openpanel/queue": "workspace:*",
     "@openpanel/redis": "workspace:*",
+    "@openpanel/validation": "workspace:*",
     "bullmq": "^5.63.0",
     "date-fns": "^3.3.1",
     "express": "^4.18.2",

apps/worker/src/boot-cron.ts

@@ -72,6 +72,11 @@ export async function bootCron() {
       type: 'flushGroups',
       pattern: 1000 * 10,
     },
+    {
+      name: 'flush',
+      type: 'flushLogs',
+      pattern: 1000 * 10,
+    },
     {
       name: 'insightsDaily',
       type: 'insightsDaily',

apps/worker/src/boot-workers.ts

@@ -8,6 +8,7 @@ import {
   gscQueue,
   importQueue,
   insightsQueue,
+  logsQueue,
   miscQueue,
   notificationQueue,
   queueLogger,
@@ -22,6 +23,7 @@ import { incomingEvent } from './jobs/events.incoming-event';
 import { gscJob } from './jobs/gsc';
 import { importJob } from './jobs/import';
 import { insightsProjectJob } from './jobs/insights';
+import { incomingLog } from './jobs/logs.incoming-log';
 import { miscJob } from './jobs/misc';
 import { notificationJob } from './jobs/notification';
 import { sessionsJob } from './jobs/sessions';
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
       'import',
       'insights',
       'gsc',
+      'logs',
     ];
   }
 
@@ -221,6 +224,22 @@ export function bootWorkers() {
     logger.info('Started worker for gsc', { concurrency });
   }
 
+  // Start logs worker
+  if (enabledQueues.includes('logs')) {
+    const concurrency = getConcurrencyFor('logs', 10);
+    const logsWorker = new Worker(logsQueue.name, async (job) => {
+      const { type, payload } = job.data;
+      if (type === 'incomingLog') {
+        return await incomingLog(payload);
+      }
+    }, {
+      ...workerOptions,
+      concurrency,
+    });
+    workers.push(logsWorker);
+    logger.info('Started worker for logs', { concurrency });
+  }
+
   if (workers.length === 0) {
     logger.warn(
       'No workers started. Check ENABLED_QUEUES environment variable.'

apps/worker/src/jobs/cron.ts

@@ -1,6 +1,7 @@
 import {
   eventBuffer,
   groupBuffer,
+  logBuffer,
   profileBackfillBuffer,
   profileBuffer,
   replayBuffer,
@@ -38,6 +39,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
     case 'flushGroups': {
       return await groupBuffer.tryFlush();
     }
+    case 'flushLogs': {
+      return await logBuffer.tryFlush();
+    }
     case 'ping': {
       return await ping();
     }

apps/worker/src/jobs/logs.incoming-log.ts

@@ -0,0 +1,63 @@
+import type { IClickhouseLog } from '@openpanel/db';
+import { logBuffer } from '@openpanel/db';
+import type { LogsQueuePayload } from '@openpanel/queue';
+import { SEVERITY_TEXT_TO_NUMBER } from '@openpanel/validation';
+import { logger as baseLogger } from '@/utils/logger';
+
+export async function incomingLog(
+  payload: LogsQueuePayload['payload'],
+): Promise<void> {
+  const logger = baseLogger.child({ projectId: payload.projectId });
+
+  try {
+    const { log, uaInfo, geo, deviceId, sessionId, projectId, headers } = payload;
+
+    const sdkName = headers['openpanel-sdk-name'] ?? '';
+    const sdkVersion = headers['openpanel-sdk-version'] ?? '';
+
+    const severityNumber =
+      log.severityNumber ??
+      SEVERITY_TEXT_TO_NUMBER[log.severity] ??
+      9; // INFO fallback
+
+    const row: IClickhouseLog = {
+      project_id: projectId,
+      device_id: deviceId,
+      profile_id: log.profileId ? String(log.profileId) : '',
+      session_id: sessionId,
+      timestamp: log.timestamp,
+      observed_at: new Date().toISOString(),
+      severity_number: severityNumber,
+      severity_text: log.severity,
+      body: log.body,
+      trace_id: log.traceId ?? '',
+      span_id: log.spanId ?? '',
+      trace_flags: log.traceFlags ?? 0,
+      logger_name: log.loggerName ?? '',
+      attributes: log.attributes ?? {},
+      resource: log.resource ?? {},
+      sdk_name: sdkName,
+      sdk_version: sdkVersion,
+      country: geo.country ?? '',
+      city: geo.city ?? '',
+      region: geo.region ?? '',
+      os: uaInfo.os ?? '',
+      os_version: uaInfo.osVersion ?? '',
+      browser: uaInfo.isServer ? '' : (uaInfo.browser ?? ''),
+      browser_version: uaInfo.isServer ? '' : (uaInfo.browserVersion ?? ''),
+      device: uaInfo.device ?? '',
+      brand: uaInfo.isServer ? '' : (uaInfo.brand ?? ''),
+      model: uaInfo.isServer ? '' : (uaInfo.model ?? ''),
+    };
+
+    logBuffer.add(row);
+
+    logger.info('Log queued', {
+      severity: log.severity,
+      loggerName: log.loggerName,
+    });
+  } catch (error) {
+    logger.error('Failed to process incoming log', { error });
+    throw error;
+  }
+}