feat:add otel logging

This commit is contained in:
2026-03-31 16:45:05 +02:00
parent fcb4cf5fb0
commit 655ea1f87e
23 changed files with 1334 additions and 1 deletions

View File

@@ -0,0 +1,68 @@
import { parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { getGeoLocation } from '@openpanel/geo';
import { type LogsQueuePayload, logsQueue } from '@openpanel/queue';
import { type ILogBatchPayload, zLogBatchPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { getDeviceId } from '@/utils/ids';
import { getStringHeaders } from './track.controller';
export async function handler(
request: FastifyRequest<{ Body: ILogBatchPayload }>,
reply: FastifyReply,
) {
const projectId = request.client?.projectId;
if (!projectId) {
return reply.status(400).send({ status: 400, error: 'Missing projectId' });
}
const validationResult = zLogBatchPayload.safeParse(request.body);
if (!validationResult.success) {
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Validation failed',
errors: validationResult.error.errors,
});
}
const { logs } = validationResult.data;
const ip = request.clientIp;
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
const headers = getStringHeaders(request.headers);
const receivedAt = new Date().toISOString();
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
const { deviceId, sessionId } = await getDeviceId({ projectId, ip, ua, salts });
const uaInfo = parseUserAgent(ua, undefined);
const jobs: LogsQueuePayload[] = logs.map((log) => ({
type: 'incomingLog' as const,
payload: {
projectId,
log: {
...log,
timestamp: log.timestamp ?? receivedAt,
},
uaInfo,
geo: {
country: geo.country,
city: geo.city,
region: geo.region,
},
headers,
deviceId,
sessionId,
},
}));
await logsQueue.addBulk(
jobs.map((job) => ({
name: 'incomingLog',
data: job,
})),
);
return reply.status(200).send({ ok: true, count: logs.length });
}

View File

@@ -40,6 +40,7 @@ import gscCallbackRouter from './routes/gsc-callback.router';
import importRouter from './routes/import.router';
import insightsRouter from './routes/insights.router';
import liveRouter from './routes/live.router';
import logsRouter from './routes/logs.router';
import manageRouter from './routes/manage.router';
import miscRouter from './routes/misc.router';
import oauthRouter from './routes/oauth-callback.router';
@@ -198,6 +199,7 @@ const startServer = async () => {
instance.register(gscCallbackRouter, { prefix: '/gsc' });
instance.register(miscRouter, { prefix: '/misc' });
instance.register(aiRouter, { prefix: '/ai' });
instance.register(logsRouter, { prefix: '/logs' });
});
// Public API

View File

@@ -0,0 +1,6 @@
import { handler } from '@/controllers/logs.controller';
import type { FastifyInstance } from 'fastify';
export default async function (fastify: FastifyInstance) {
fastify.post('/', handler);
}

View File

@@ -15,6 +15,7 @@ import {
LayoutDashboardIcon,
LayoutPanelTopIcon,
PlusIcon,
ScrollTextIcon,
SearchIcon,
SparklesIcon,
TrendingUpDownIcon,
@@ -61,6 +62,7 @@ export default function SidebarProjectMenu({
<SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" />
<SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" />
<SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" />
<SidebarLink href={'/logs'} icon={ScrollTextIcon} label="Logs" />
<SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" />
<SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" />
<SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" />

View File

@@ -47,6 +47,7 @@ import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/
import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references'
import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime'
import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages'
import { Route as AppOrganizationIdProjectIdLogsRouteImport } from './routes/_app.$organizationId.$projectId.logs'
import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights'
import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups'
import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards'
@@ -352,6 +353,12 @@ const AppOrganizationIdProjectIdPagesRoute =
path: '/pages',
getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any)
const AppOrganizationIdProjectIdLogsRoute =
AppOrganizationIdProjectIdLogsRouteImport.update({
id: '/logs',
path: '/logs',
getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any)
const AppOrganizationIdProjectIdInsightsRoute =
AppOrganizationIdProjectIdInsightsRouteImport.update({
id: '/insights',
@@ -660,6 +667,7 @@ export interface FileRoutesByFullPath {
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -738,6 +746,7 @@ export interface FileRoutesByTo {
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -814,6 +823,7 @@ export interface FileRoutesById {
'/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/_app/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -905,6 +915,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/dashboards'
| '/$organizationId/$projectId/groups'
| '/$organizationId/$projectId/insights'
| '/$organizationId/$projectId/logs'
| '/$organizationId/$projectId/pages'
| '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references'
@@ -983,6 +994,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/dashboards'
| '/$organizationId/$projectId/groups'
| '/$organizationId/$projectId/insights'
| '/$organizationId/$projectId/logs'
| '/$organizationId/$projectId/pages'
| '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references'
@@ -1058,6 +1070,7 @@ export interface FileRouteTypes {
| '/_app/$organizationId/$projectId/dashboards'
| '/_app/$organizationId/$projectId/groups'
| '/_app/$organizationId/$projectId/insights'
| '/_app/$organizationId/$projectId/logs'
| '/_app/$organizationId/$projectId/pages'
| '/_app/$organizationId/$projectId/realtime'
| '/_app/$organizationId/$projectId/references'
@@ -1444,6 +1457,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute
}
'/_app/$organizationId/$projectId/logs': {
id: '/_app/$organizationId/$projectId/logs'
path: '/logs'
fullPath: '/$organizationId/$projectId/logs'
preLoaderRoute: typeof AppOrganizationIdProjectIdLogsRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute
}
'/_app/$organizationId/$projectId/insights': {
id: '/_app/$organizationId/$projectId/insights'
path: '/insights'
@@ -2028,6 +2048,7 @@ interface AppOrganizationIdProjectIdRouteChildren {
AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute
AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute
AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute
AppOrganizationIdProjectIdLogsRoute: typeof AppOrganizationIdProjectIdLogsRoute
AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute
AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute
AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute
@@ -2054,6 +2075,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh
AppOrganizationIdProjectIdGroupsRoute,
AppOrganizationIdProjectIdInsightsRoute:
AppOrganizationIdProjectIdInsightsRoute,
AppOrganizationIdProjectIdLogsRoute: AppOrganizationIdProjectIdLogsRoute,
AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute,
AppOrganizationIdProjectIdRealtimeRoute:
AppOrganizationIdProjectIdRealtimeRoute,

View File

@@ -0,0 +1,385 @@
import { useInfiniteQuery, useQuery } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import { format } from 'date-fns';
import { AnimatePresence, motion } from 'framer-motion';
import {
AlertCircleIcon,
AlertTriangleIcon,
BugIcon,
ChevronDownIcon,
ChevronRightIcon,
InfoIcon,
ScrollTextIcon,
SearchIcon,
SkullIcon,
TerminalIcon,
XIcon,
} from 'lucide-react';
import { useMemo, useState } from 'react';
import { PageContainer } from '@/components/page-container';
import { PageHeader } from '@/components/page-header';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { useSearchQueryState } from '@/hooks/use-search-query-state';
import { useTRPC } from '@/integrations/trpc/react';
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
import type { ISeverityText } from '@openpanel/validation';
export const Route = createFileRoute(
'/_app/$organizationId/$projectId/logs'
)({
component: Component,
head: () => {
return {
meta: [
{
title: createProjectTitle(PAGE_TITLES.LOGS),
},
],
};
},
});
const SEVERITY_ICONS: Record<ISeverityText, typeof InfoIcon> = {
trace: TerminalIcon,
debug: BugIcon,
info: InfoIcon,
warn: AlertTriangleIcon,
warning: AlertTriangleIcon,
error: AlertCircleIcon,
fatal: SkullIcon,
critical: SkullIcon,
};
const SEVERITY_COLORS: Record<ISeverityText, string> = {
trace: 'text-gray-400',
debug: 'text-blue-400',
info: 'text-green-400',
warn: 'text-yellow-400',
warning: 'text-yellow-400',
error: 'text-red-400',
fatal: 'text-red-600',
critical: 'text-red-600',
};
const SEVERITY_BG_COLORS: Record<ISeverityText, string> = {
trace: 'bg-gray-500/10 hover:bg-gray-500/20',
debug: 'bg-blue-500/10 hover:bg-blue-500/20',
info: 'bg-green-500/10 hover:bg-green-500/20',
warn: 'bg-yellow-500/10 hover:bg-yellow-500/20',
warning: 'bg-yellow-500/10 hover:bg-yellow-500/20',
error: 'bg-red-500/10 hover:bg-red-500/20',
fatal: 'bg-red-600/10 hover:bg-red-600/20',
critical: 'bg-red-600/10 hover:bg-red-600/20',
};
function Component() {
const { projectId } = Route.useParams();
const trpc = useTRPC();
const { search, setSearch, debouncedSearch } = useSearchQueryState();
const [selectedSeverity, setSelectedSeverity] = useState<ISeverityText[]>([]);
const [expandedLog, setExpandedLog] = useState<string | null>(null);
const severityCountsQuery = useQuery(
trpc.log.severityCounts.queryOptions({ projectId }, { enabled: !!projectId })
);
const logsQuery = useInfiniteQuery(
trpc.log.list.infiniteQueryOptions(
{
projectId,
take: 50,
search: debouncedSearch || undefined,
severity: selectedSeverity.length > 0 ? selectedSeverity : undefined,
},
{
getNextPageParam: (lastPage) => lastPage.meta.next,
}
)
);
const logs = useMemo(() => {
return logsQuery.data?.pages.flatMap((page) => page.data) ?? [];
}, [logsQuery.data]);
const severityCounts = severityCountsQuery.data ?? {};
const severityOptions: ISeverityText[] = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
const toggleSeverity = (severity: ISeverityText) => {
setSelectedSeverity((prev) =>
prev.includes(severity)
? prev.filter((s) => s !== severity)
: [...prev, severity]
);
};
return (
<PageContainer>
<PageHeader
className="mb-8"
description="View and search device and application logs"
icon={ScrollTextIcon}
title="Logs"
/>
{/* Severity Filter Chips */}
<div className="mb-6 flex flex-wrap items-center gap-2">
{severityOptions.map((severity) => {
const count = severityCounts[severity] ?? 0;
const isSelected = selectedSeverity.includes(severity);
const Icon = SEVERITY_ICONS[severity];
return (
<Button
key={severity}
className={`gap-2 capitalize ${
isSelected ? 'ring-2 ring-primary ring-offset-2' : ''
}`}
onClick={() => toggleSeverity(severity)}
size="sm"
variant="outline"
>
<Icon className={`h-4 w-4 ${SEVERITY_COLORS[severity]}`} />
<span className="capitalize">{severity}</span>
{count > 0 && (
<Badge className="ml-1" variant="secondary">
{count.toLocaleString()}
</Badge>
)}
</Button>
);
})}
{selectedSeverity.length > 0 && (
<Button
className="gap-2"
onClick={() => setSelectedSeverity([])}
size="sm"
variant="ghost"
>
<XIcon className="h-4 w-4" />
Clear filters
</Button>
)}
</div>
{/* Search */}
<div className="relative mb-6">
<SearchIcon className="absolute top-1/2 left-3 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
<Input
className="pl-10"
onChange={(e) => setSearch(e.target.value)}
placeholder="Search logs..."
value={search}
/>
</div>
{/* Logs List */}
<div className="space-y-2">
{logs.map((log) => {
const isExpanded = expandedLog === log.id;
const Icon = SEVERITY_ICONS[log.severityText as ISeverityText] ?? InfoIcon;
const severityColor = SEVERITY_COLORS[log.severityText as ISeverityText] ?? 'text-gray-400';
const bgColor = SEVERITY_BG_COLORS[log.severityText as ISeverityText] ?? 'bg-gray-500/10';
return (
<motion.div
key={log.id}
animate={{ opacity: 1, y: 0 }}
className={`rounded-lg border ${bgColor} transition-colors`}
initial={{ opacity: 0, y: 10 }}
layout
>
<button
className="flex w-full items-start gap-3 p-4 text-left"
onClick={() => setExpandedLog(isExpanded ? null : log.id)}
type="button"
>
{isExpanded ? (
<ChevronDownIcon className="mt-1 h-4 w-4 shrink-0 text-muted-foreground" />
) : (
<ChevronRightIcon className="mt-1 h-4 w-4 shrink-0 text-muted-foreground" />
)}
<Icon className={`mt-1 h-4 w-4 shrink-0 ${severityColor}`} />
<div className="min-w-0 flex-1">
<div className="flex items-center gap-2">
<span className={`font-mono text-xs uppercase ${severityColor}`}>
{log.severityText}
</span>
<span className="text-muted-foreground text-xs">
{format(new Date(log.timestamp), 'MMM d, HH:mm:ss.SSS')}
</span>
{log.loggerName && (
<Badge className="text-xs" variant="outline">
{log.loggerName}
</Badge>
)}
</div>
<p className="mt-1 truncate font-mono text-sm">{log.body}</p>
</div>
</button>
<AnimatePresence>
{isExpanded && (
<motion.div
animate={{ height: 'auto', opacity: 1 }}
className="border-t px-4 pb-4"
exit={{ height: 0, opacity: 0 }}
initial={{ height: 0, opacity: 0 }}
>
<div className="space-y-4 pt-4">
{/* Full Message */}
<div>
<h4 className="mb-2 font-medium text-sm">Message</h4>
<pre className="max-h-40 overflow-auto whitespace-pre-wrap rounded bg-muted p-3 font-mono text-sm">
{log.body}
</pre>
</div>
{/* Attributes */}
{Object.keys(log.attributes).length > 0 && (
<div>
<h4 className="mb-2 font-medium text-sm">Attributes</h4>
<div className="grid gap-2">
{Object.entries(log.attributes).map(([key, value]) => (
<div
key={key}
className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2"
>
<span className="font-mono text-muted-foreground text-sm">
{key}
</span>
<span className="font-mono text-sm">{value}</span>
</div>
))}
</div>
</div>
)}
{/* Resource */}
{Object.keys(log.resource).length > 0 && (
<div>
<h4 className="mb-2 font-medium text-sm">Resource</h4>
<div className="grid gap-2">
{Object.entries(log.resource).map(([key, value]) => (
<div
key={key}
className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2"
>
<span className="font-mono text-muted-foreground text-sm">
{key}
</span>
<span className="font-mono text-sm">{value}</span>
</div>
))}
</div>
</div>
)}
{/* Trace Context */}
{(log.traceId || log.spanId) && (
<div>
<h4 className="mb-2 font-medium text-sm">Trace Context</h4>
<div className="space-y-2">
{log.traceId && (
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="font-mono text-muted-foreground text-sm">
Trace ID
</span>
<span className="font-mono text-sm">{log.traceId}</span>
</div>
)}
{log.spanId && (
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="font-mono text-muted-foreground text-sm">
Span ID
</span>
<span className="font-mono text-sm">{log.spanId}</span>
</div>
)}
</div>
</div>
)}
{/* Device Info */}
<div>
<h4 className="mb-2 font-medium text-sm">Device</h4>
<div className="grid gap-2 text-sm">
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="text-muted-foreground">Device ID</span>
<span className="font-mono">{log.deviceId}</span>
</div>
{log.profileId && (
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="text-muted-foreground">Profile ID</span>
<span className="font-mono">{log.profileId}</span>
</div>
)}
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="text-muted-foreground">OS</span>
<span>
{log.os} {log.osVersion}
</span>
</div>
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="text-muted-foreground">Browser</span>
<span>
{log.browser} {log.browserVersion}
</span>
</div>
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="text-muted-foreground">Location</span>
<span>
{log.city}, {log.region}, {log.country}
</span>
</div>
</div>
</div>
{/* SDK Info */}
<div>
<h4 className="mb-2 font-medium text-sm">SDK</h4>
<div className="grid gap-2 text-sm">
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="text-muted-foreground">Name</span>
<span>{log.sdkName || 'unknown'}</span>
</div>
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
<span className="text-muted-foreground">Version</span>
<span>{log.sdkVersion || 'unknown'}</span>
</div>
</div>
</div>
{/* Observed At */}
<div className="text-muted-foreground text-xs">
Observed at: {format(new Date(log.observedAt), 'MMM d, HH:mm:ss.SSS')}
</div>
</div>
</motion.div>
)}
</AnimatePresence>
</motion.div>
);
})}
</div>
{/* Load More */}
{logsQuery.hasNextPage && (
<div className="mt-6 flex justify-center">
<Button
disabled={logsQuery.isFetchingNextPage}
onClick={() => logsQuery.fetchNextPage()}
variant="outline"
>
{logsQuery.isFetchingNextPage ? 'Loading...' : 'Load more'}
</Button>
</div>
)}
</PageContainer>
);
}

View File

@@ -97,6 +97,8 @@ export const PAGE_TITLES = {
PROFILE_DETAILS: 'Profile details',
// Groups
GROUPS: 'Groups',
// Logs
LOGS: 'Logs',
GROUP_DETAILS: 'Group details',
// Sub-sections

View File

@@ -24,6 +24,7 @@
"@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",
"@openpanel/validation": "workspace:*",
"bullmq": "^5.63.0",
"date-fns": "^3.3.1",
"express": "^4.18.2",

View File

@@ -72,6 +72,11 @@ export async function bootCron() {
type: 'flushGroups',
pattern: 1000 * 10,
},
{
name: 'flush',
type: 'flushLogs',
pattern: 1000 * 10,
},
{
name: 'insightsDaily',
type: 'insightsDaily',

View File

@@ -8,6 +8,7 @@ import {
gscQueue,
importQueue,
insightsQueue,
logsQueue,
miscQueue,
notificationQueue,
queueLogger,
@@ -22,6 +23,7 @@ import { incomingEvent } from './jobs/events.incoming-event';
import { gscJob } from './jobs/gsc';
import { importJob } from './jobs/import';
import { insightsProjectJob } from './jobs/insights';
import { incomingLog } from './jobs/logs.incoming-log';
import { miscJob } from './jobs/misc';
import { notificationJob } from './jobs/notification';
import { sessionsJob } from './jobs/sessions';
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
'import',
'insights',
'gsc',
'logs',
];
}
@@ -221,6 +224,22 @@ export function bootWorkers() {
logger.info('Started worker for gsc', { concurrency });
}
// Start logs worker
if (enabledQueues.includes('logs')) {
const concurrency = getConcurrencyFor('logs', 10);
const logsWorker = new Worker(logsQueue.name, async (job) => {
const { type, payload } = job.data;
if (type === 'incomingLog') {
return await incomingLog(payload);
}
}, {
...workerOptions,
concurrency,
});
workers.push(logsWorker);
logger.info('Started worker for logs', { concurrency });
}
if (workers.length === 0) {
logger.warn(
'No workers started. Check ENABLED_QUEUES environment variable.'

View File

@@ -1,6 +1,7 @@
import {
eventBuffer,
groupBuffer,
logBuffer,
profileBackfillBuffer,
profileBuffer,
replayBuffer,
@@ -38,6 +39,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
case 'flushGroups': {
return await groupBuffer.tryFlush();
}
case 'flushLogs': {
return await logBuffer.tryFlush();
}
case 'ping': {
return await ping();
}

View File

@@ -0,0 +1,63 @@
import type { IClickhouseLog } from '@openpanel/db';
import { logBuffer } from '@openpanel/db';
import type { LogsQueuePayload } from '@openpanel/queue';
import { SEVERITY_TEXT_TO_NUMBER } from '@openpanel/validation';
import { logger as baseLogger } from '@/utils/logger';
export async function incomingLog(
payload: LogsQueuePayload['payload'],
): Promise<void> {
const logger = baseLogger.child({ projectId: payload.projectId });
try {
const { log, uaInfo, geo, deviceId, sessionId, projectId, headers } = payload;
const sdkName = headers['openpanel-sdk-name'] ?? '';
const sdkVersion = headers['openpanel-sdk-version'] ?? '';
const severityNumber =
log.severityNumber ??
SEVERITY_TEXT_TO_NUMBER[log.severity] ??
9; // INFO fallback
const row: IClickhouseLog = {
project_id: projectId,
device_id: deviceId,
profile_id: log.profileId ? String(log.profileId) : '',
session_id: sessionId,
timestamp: log.timestamp,
observed_at: new Date().toISOString(),
severity_number: severityNumber,
severity_text: log.severity,
body: log.body,
trace_id: log.traceId ?? '',
span_id: log.spanId ?? '',
trace_flags: log.traceFlags ?? 0,
logger_name: log.loggerName ?? '',
attributes: log.attributes ?? {},
resource: log.resource ?? {},
sdk_name: sdkName,
sdk_version: sdkVersion,
country: geo.country ?? '',
city: geo.city ?? '',
region: geo.region ?? '',
os: uaInfo.os ?? '',
os_version: uaInfo.osVersion ?? '',
browser: uaInfo.isServer ? '' : (uaInfo.browser ?? ''),
browser_version: uaInfo.isServer ? '' : (uaInfo.browserVersion ?? ''),
device: uaInfo.device ?? '',
brand: uaInfo.isServer ? '' : (uaInfo.brand ?? ''),
model: uaInfo.isServer ? '' : (uaInfo.model ?? ''),
};
logBuffer.add(row);
logger.info('Log queued', {
severity: log.severity,
loggerName: log.loggerName,
});
} catch (error) {
logger.error('Failed to process incoming log', { error });
throw error;
}
}