feat: add OpenTelemetry device log capture pipeline
- ClickHouse `logs` table (migration 13) with OTel columns, bloom filter indices - Zod validation schema for log payloads (severity, body, attributes, trace context) - Redis-backed LogBuffer with micro-batching into ClickHouse - POST /logs API endpoint with client auth, geo + UA enrichment - BullMQ logs queue + worker job - cron flushLogs every 10s wired into existing cron system - SDK captureLog(severity, body, properties) with client-side batching Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
68
apps/api/src/controllers/logs.controller.ts
Normal file
68
apps/api/src/controllers/logs.controller.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { parseUserAgent } from '@openpanel/common/server';
|
||||
import { getSalts } from '@openpanel/db';
|
||||
import { getGeoLocation } from '@openpanel/geo';
|
||||
import { type LogsQueuePayload, logsQueue } from '@openpanel/queue';
|
||||
import { type ILogBatchPayload, zLogBatchPayload } from '@openpanel/validation';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { getDeviceId } from '@/utils/ids';
|
||||
import { getStringHeaders } from './track.controller';
|
||||
|
||||
export async function handler(
|
||||
request: FastifyRequest<{ Body: ILogBatchPayload }>,
|
||||
reply: FastifyReply,
|
||||
) {
|
||||
const projectId = request.client?.projectId;
|
||||
if (!projectId) {
|
||||
return reply.status(400).send({ status: 400, error: 'Missing projectId' });
|
||||
}
|
||||
|
||||
const validationResult = zLogBatchPayload.safeParse(request.body);
|
||||
if (!validationResult.success) {
|
||||
return reply.status(400).send({
|
||||
status: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Validation failed',
|
||||
errors: validationResult.error.errors,
|
||||
});
|
||||
}
|
||||
|
||||
const { logs } = validationResult.data;
|
||||
|
||||
const ip = request.clientIp;
|
||||
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
|
||||
const headers = getStringHeaders(request.headers);
|
||||
const receivedAt = new Date().toISOString();
|
||||
|
||||
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
|
||||
const { deviceId, sessionId } = await getDeviceId({ projectId, ip, ua, salts });
|
||||
const uaInfo = parseUserAgent(ua, undefined);
|
||||
|
||||
const jobs: LogsQueuePayload[] = logs.map((log) => ({
|
||||
type: 'incomingLog' as const,
|
||||
payload: {
|
||||
projectId,
|
||||
log: {
|
||||
...log,
|
||||
timestamp: log.timestamp ?? receivedAt,
|
||||
},
|
||||
uaInfo,
|
||||
geo: {
|
||||
country: geo.country,
|
||||
city: geo.city,
|
||||
region: geo.region,
|
||||
},
|
||||
headers,
|
||||
deviceId,
|
||||
sessionId,
|
||||
},
|
||||
}));
|
||||
|
||||
await logsQueue.addBulk(
|
||||
jobs.map((job) => ({
|
||||
name: 'incomingLog',
|
||||
data: job,
|
||||
})),
|
||||
);
|
||||
|
||||
return reply.status(200).send({ ok: true, count: logs.length });
|
||||
}
|
||||
Reference in New Issue
Block a user