feat: add OpenTelemetry device log capture pipeline

- ClickHouse `logs` table (migration 13) with OTel columns, bloom filter indices
- Zod validation schema for log payloads (severity, body, attributes, trace context)
- Redis-backed LogBuffer with micro-batching into ClickHouse
- POST /logs API endpoint with client auth, geo + UA enrichment
- BullMQ logs queue + worker job
- cron flushLogs every 10s wired into existing cron system
- SDK captureLog(severity, body, properties) with client-side batching

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-30 12:04:04 +02:00
parent a1ce71ffb6
commit 0672857974
14 changed files with 652 additions and 2 deletions

View File

@@ -7,6 +7,7 @@ import type {
IGroupPayload as GroupPayload,
IIdentifyPayload as IdentifyPayload,
IIncrementPayload as IncrementPayload,
ISeverityText,
ITrackHandlerPayload as TrackHandlerPayload,
ITrackPayload as TrackPayload,
} from '@openpanel/validation';
@@ -19,6 +20,7 @@ export type {
GroupPayload,
IdentifyPayload,
IncrementPayload,
ISeverityText,
TrackHandlerPayload,
TrackPayload,
};
@@ -29,6 +31,33 @@ export interface TrackProperties {
groups?: string[];
}
export interface LogProperties {
/** Logger name (e.g. "com.example.MyActivity") */
loggerName?: string;
traceId?: string;
spanId?: string;
traceFlags?: number;
/** Log-level key-value attributes */
attributes?: Record<string, string>;
/** Resource/device attributes */
resource?: Record<string, string>;
/** ISO 8601 timestamp; defaults to now */
timestamp?: string;
}
interface LogPayloadForQueue {
body: string;
severity: ISeverityText;
loggerName?: string;
traceId?: string;
spanId?: string;
traceFlags?: number;
attributes?: Record<string, string>;
resource?: Record<string, string>;
timestamp: string;
profileId?: string;
}
export type UpsertGroupPayload = GroupPayload;
export interface OpenPanelOptions {
@@ -57,6 +86,10 @@ export class OpenPanel {
sessionId?: string;
global?: Record<string, unknown>;
queue: TrackHandlerPayload[] = [];
private logQueue: LogPayloadForQueue[] = [];
private logFlushTimer: ReturnType<typeof setTimeout> | null = null;
private logFlushIntervalMs = 2000;
private logFlushMaxSize = 50;
constructor(options: OpenPanelOptions) {
this.options = options;
@@ -327,6 +360,67 @@ export class OpenPanel {
this.queue = remaining;
}
captureLog(
severity: ISeverityText,
body: string,
properties?: LogProperties,
) {
if (this.options.disabled) {
return;
}
const entry: LogPayloadForQueue = {
body,
severity,
timestamp: properties?.timestamp ?? new Date().toISOString(),
...(this.profileId ? { profileId: this.profileId } : {}),
...(properties?.loggerName ? { loggerName: properties.loggerName } : {}),
...(properties?.traceId ? { traceId: properties.traceId } : {}),
...(properties?.spanId ? { spanId: properties.spanId } : {}),
...(properties?.traceFlags !== undefined
? { traceFlags: properties.traceFlags }
: {}),
...(properties?.attributes ? { attributes: properties.attributes } : {}),
...(properties?.resource ? { resource: properties.resource } : {}),
};
this.logQueue.push(entry);
if (this.logQueue.length >= this.logFlushMaxSize) {
this.flushLogs();
return;
}
if (!this.logFlushTimer) {
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null;
this.flushLogs();
}, this.logFlushIntervalMs);
}
}
private async flushLogs() {
if (this.logFlushTimer) {
clearTimeout(this.logFlushTimer);
this.logFlushTimer = null;
}
if (this.logQueue.length === 0) {
return;
}
const batch = this.logQueue;
this.logQueue = [];
try {
await this.api.fetch('/logs', { logs: batch });
} catch (error) {
this.log('Failed to flush logs', error);
// Re-queue on failure
this.logQueue = batch.concat(this.logQueue);
}
}
log(...args: any[]) {
if (this.options.debug) {
console.log('[OpenPanel.dev]', ...args);