feat: add OpenTelemetry device log capture pipeline
- ClickHouse `logs` table (migration 13) with OTel columns, bloom filter indices - Zod validation schema for log payloads (severity, body, attributes, trace context) - Redis-backed LogBuffer with micro-batching into ClickHouse - POST /logs API endpoint with client auth, geo + UA enrichment - BullMQ logs queue + worker job - cron flushLogs every 10s wired into existing cron system - SDK captureLog(severity, body, properties) with client-side batching Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,7 @@ import { createLogger } from '@openpanel/logger';
|
||||
import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
|
||||
import { Queue } from 'bullmq';
|
||||
import { Queue as GroupQueue } from 'groupmq';
|
||||
import type { ITrackPayload } from '../../validation';
|
||||
import type { ILogPayload, ITrackPayload } from '../../validation';
|
||||
|
||||
export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(
|
||||
process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',
|
||||
@@ -138,6 +138,10 @@ export type CronQueuePayloadFlushGroups = {
|
||||
type: 'flushGroups';
|
||||
payload: undefined;
|
||||
};
|
||||
export type CronQueuePayloadFlushLogs = {
|
||||
type: 'flushLogs';
|
||||
payload: undefined;
|
||||
};
|
||||
export type CronQueuePayload =
|
||||
| CronQueuePayloadSalt
|
||||
| CronQueuePayloadFlushEvents
|
||||
@@ -146,6 +150,7 @@ export type CronQueuePayload =
|
||||
| CronQueuePayloadFlushProfileBackfill
|
||||
| CronQueuePayloadFlushReplay
|
||||
| CronQueuePayloadFlushGroups
|
||||
| CronQueuePayloadFlushLogs
|
||||
| CronQueuePayloadPing
|
||||
| CronQueuePayloadProject
|
||||
| CronQueuePayloadInsightsDaily
|
||||
@@ -297,3 +302,50 @@ export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {
|
||||
removeOnFail: 100,
|
||||
},
|
||||
});
|
||||
|
||||
export type LogsQueuePayload = {
|
||||
type: 'incomingLog';
|
||||
payload: {
|
||||
projectId: string;
|
||||
log: ILogPayload & {
|
||||
timestamp: string;
|
||||
};
|
||||
uaInfo:
|
||||
| {
|
||||
readonly isServer: true;
|
||||
readonly device: 'server';
|
||||
readonly os: '';
|
||||
readonly osVersion: '';
|
||||
readonly browser: '';
|
||||
readonly browserVersion: '';
|
||||
readonly brand: '';
|
||||
readonly model: '';
|
||||
}
|
||||
| {
|
||||
readonly os: string | undefined;
|
||||
readonly osVersion: string | undefined;
|
||||
readonly browser: string | undefined;
|
||||
readonly browserVersion: string | undefined;
|
||||
readonly device: string;
|
||||
readonly brand: string | undefined;
|
||||
readonly model: string | undefined;
|
||||
readonly isServer: false;
|
||||
};
|
||||
geo: {
|
||||
country: string | undefined;
|
||||
city: string | undefined;
|
||||
region: string | undefined;
|
||||
};
|
||||
headers: Record<string, string | undefined>;
|
||||
deviceId: string;
|
||||
sessionId: string;
|
||||
};
|
||||
};
|
||||
|
||||
export const logsQueue = new Queue<LogsQueuePayload>(getQueueName('logs'), {
|
||||
connection: getRedisQueue(),
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 1000,
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user