feat: add OpenTelemetry device log capture pipeline
- ClickHouse `logs` table (migration 13) with OTel columns, bloom filter indices - Zod validation schema for log payloads (severity, body, attributes, trace context) - Redis-backed LogBuffer with micro-batching into ClickHouse - POST /logs API endpoint with client auth, geo + UA enrichment - BullMQ logs queue + worker job - cron flushLogs every 10s wired into existing cron system - SDK captureLog(severity, body, properties) with client-side batching Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import type {
|
||||
IGroupPayload as GroupPayload,
|
||||
IIdentifyPayload as IdentifyPayload,
|
||||
IIncrementPayload as IncrementPayload,
|
||||
ISeverityText,
|
||||
ITrackHandlerPayload as TrackHandlerPayload,
|
||||
ITrackPayload as TrackPayload,
|
||||
} from '@openpanel/validation';
|
||||
@@ -19,6 +20,7 @@ export type {
|
||||
GroupPayload,
|
||||
IdentifyPayload,
|
||||
IncrementPayload,
|
||||
ISeverityText,
|
||||
TrackHandlerPayload,
|
||||
TrackPayload,
|
||||
};
|
||||
@@ -29,6 +31,33 @@ export interface TrackProperties {
|
||||
groups?: string[];
|
||||
}
|
||||
|
||||
export interface LogProperties {
|
||||
/** Logger name (e.g. "com.example.MyActivity") */
|
||||
loggerName?: string;
|
||||
traceId?: string;
|
||||
spanId?: string;
|
||||
traceFlags?: number;
|
||||
/** Log-level key-value attributes */
|
||||
attributes?: Record<string, string>;
|
||||
/** Resource/device attributes */
|
||||
resource?: Record<string, string>;
|
||||
/** ISO 8601 timestamp; defaults to now */
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
interface LogPayloadForQueue {
|
||||
body: string;
|
||||
severity: ISeverityText;
|
||||
loggerName?: string;
|
||||
traceId?: string;
|
||||
spanId?: string;
|
||||
traceFlags?: number;
|
||||
attributes?: Record<string, string>;
|
||||
resource?: Record<string, string>;
|
||||
timestamp: string;
|
||||
profileId?: string;
|
||||
}
|
||||
|
||||
export type UpsertGroupPayload = GroupPayload;
|
||||
|
||||
export interface OpenPanelOptions {
|
||||
@@ -57,6 +86,10 @@ export class OpenPanel {
|
||||
sessionId?: string;
|
||||
global?: Record<string, unknown>;
|
||||
queue: TrackHandlerPayload[] = [];
|
||||
private logQueue: LogPayloadForQueue[] = [];
|
||||
private logFlushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private logFlushIntervalMs = 2000;
|
||||
private logFlushMaxSize = 50;
|
||||
|
||||
constructor(options: OpenPanelOptions) {
|
||||
this.options = options;
|
||||
@@ -327,6 +360,67 @@ export class OpenPanel {
|
||||
this.queue = remaining;
|
||||
}
|
||||
|
||||
captureLog(
|
||||
severity: ISeverityText,
|
||||
body: string,
|
||||
properties?: LogProperties,
|
||||
) {
|
||||
if (this.options.disabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: LogPayloadForQueue = {
|
||||
body,
|
||||
severity,
|
||||
timestamp: properties?.timestamp ?? new Date().toISOString(),
|
||||
...(this.profileId ? { profileId: this.profileId } : {}),
|
||||
...(properties?.loggerName ? { loggerName: properties.loggerName } : {}),
|
||||
...(properties?.traceId ? { traceId: properties.traceId } : {}),
|
||||
...(properties?.spanId ? { spanId: properties.spanId } : {}),
|
||||
...(properties?.traceFlags !== undefined
|
||||
? { traceFlags: properties.traceFlags }
|
||||
: {}),
|
||||
...(properties?.attributes ? { attributes: properties.attributes } : {}),
|
||||
...(properties?.resource ? { resource: properties.resource } : {}),
|
||||
};
|
||||
|
||||
this.logQueue.push(entry);
|
||||
|
||||
if (this.logQueue.length >= this.logFlushMaxSize) {
|
||||
this.flushLogs();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.logFlushTimer) {
|
||||
this.logFlushTimer = setTimeout(() => {
|
||||
this.logFlushTimer = null;
|
||||
this.flushLogs();
|
||||
}, this.logFlushIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private async flushLogs() {
|
||||
if (this.logFlushTimer) {
|
||||
clearTimeout(this.logFlushTimer);
|
||||
this.logFlushTimer = null;
|
||||
}
|
||||
|
||||
if (this.logQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const batch = this.logQueue;
|
||||
this.logQueue = [];
|
||||
|
||||
try {
|
||||
await this.api.fetch('/logs', { logs: batch });
|
||||
} catch (error) {
|
||||
this.log('Failed to flush logs', error);
|
||||
// Re-queue on failure
|
||||
this.logQueue = batch.concat(this.logQueue);
|
||||
}
|
||||
}
|
||||
|
||||
log(...args: any[]) {
|
||||
if (this.options.debug) {
|
||||
console.log('[OpenPanel.dev]', ...args);
|
||||
|
||||
Reference in New Issue
Block a user