feat:add otel logging
This commit is contained in:
68
apps/api/src/controllers/logs.controller.ts
Normal file
68
apps/api/src/controllers/logs.controller.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { parseUserAgent } from '@openpanel/common/server';
|
||||
import { getSalts } from '@openpanel/db';
|
||||
import { getGeoLocation } from '@openpanel/geo';
|
||||
import { type LogsQueuePayload, logsQueue } from '@openpanel/queue';
|
||||
import { type ILogBatchPayload, zLogBatchPayload } from '@openpanel/validation';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { getDeviceId } from '@/utils/ids';
|
||||
import { getStringHeaders } from './track.controller';
|
||||
|
||||
export async function handler(
|
||||
request: FastifyRequest<{ Body: ILogBatchPayload }>,
|
||||
reply: FastifyReply,
|
||||
) {
|
||||
const projectId = request.client?.projectId;
|
||||
if (!projectId) {
|
||||
return reply.status(400).send({ status: 400, error: 'Missing projectId' });
|
||||
}
|
||||
|
||||
const validationResult = zLogBatchPayload.safeParse(request.body);
|
||||
if (!validationResult.success) {
|
||||
return reply.status(400).send({
|
||||
status: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Validation failed',
|
||||
errors: validationResult.error.errors,
|
||||
});
|
||||
}
|
||||
|
||||
const { logs } = validationResult.data;
|
||||
|
||||
const ip = request.clientIp;
|
||||
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
|
||||
const headers = getStringHeaders(request.headers);
|
||||
const receivedAt = new Date().toISOString();
|
||||
|
||||
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
|
||||
const { deviceId, sessionId } = await getDeviceId({ projectId, ip, ua, salts });
|
||||
const uaInfo = parseUserAgent(ua, undefined);
|
||||
|
||||
const jobs: LogsQueuePayload[] = logs.map((log) => ({
|
||||
type: 'incomingLog' as const,
|
||||
payload: {
|
||||
projectId,
|
||||
log: {
|
||||
...log,
|
||||
timestamp: log.timestamp ?? receivedAt,
|
||||
},
|
||||
uaInfo,
|
||||
geo: {
|
||||
country: geo.country,
|
||||
city: geo.city,
|
||||
region: geo.region,
|
||||
},
|
||||
headers,
|
||||
deviceId,
|
||||
sessionId,
|
||||
},
|
||||
}));
|
||||
|
||||
await logsQueue.addBulk(
|
||||
jobs.map((job) => ({
|
||||
name: 'incomingLog',
|
||||
data: job,
|
||||
})),
|
||||
);
|
||||
|
||||
return reply.status(200).send({ ok: true, count: logs.length });
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import gscCallbackRouter from './routes/gsc-callback.router';
|
||||
import importRouter from './routes/import.router';
|
||||
import insightsRouter from './routes/insights.router';
|
||||
import liveRouter from './routes/live.router';
|
||||
import logsRouter from './routes/logs.router';
|
||||
import manageRouter from './routes/manage.router';
|
||||
import miscRouter from './routes/misc.router';
|
||||
import oauthRouter from './routes/oauth-callback.router';
|
||||
@@ -198,6 +199,7 @@ const startServer = async () => {
|
||||
instance.register(gscCallbackRouter, { prefix: '/gsc' });
|
||||
instance.register(miscRouter, { prefix: '/misc' });
|
||||
instance.register(aiRouter, { prefix: '/ai' });
|
||||
instance.register(logsRouter, { prefix: '/logs' });
|
||||
});
|
||||
|
||||
// Public API
|
||||
|
||||
6
apps/api/src/routes/logs.router.ts
Normal file
6
apps/api/src/routes/logs.router.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
import { handler } from '@/controllers/logs.controller';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
export default async function (fastify: FastifyInstance) {
|
||||
fastify.post('/', handler);
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
LayoutDashboardIcon,
|
||||
LayoutPanelTopIcon,
|
||||
PlusIcon,
|
||||
ScrollTextIcon,
|
||||
SearchIcon,
|
||||
SparklesIcon,
|
||||
TrendingUpDownIcon,
|
||||
@@ -61,6 +62,7 @@ export default function SidebarProjectMenu({
|
||||
<SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" />
|
||||
<SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" />
|
||||
<SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" />
|
||||
<SidebarLink href={'/logs'} icon={ScrollTextIcon} label="Logs" />
|
||||
<SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" />
|
||||
<SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" />
|
||||
<SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" />
|
||||
|
||||
@@ -47,6 +47,7 @@ import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/
|
||||
import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references'
|
||||
import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime'
|
||||
import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages'
|
||||
import { Route as AppOrganizationIdProjectIdLogsRouteImport } from './routes/_app.$organizationId.$projectId.logs'
|
||||
import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights'
|
||||
import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups'
|
||||
import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards'
|
||||
@@ -352,6 +353,12 @@ const AppOrganizationIdProjectIdPagesRoute =
|
||||
path: '/pages',
|
||||
getParentRoute: () => AppOrganizationIdProjectIdRoute,
|
||||
} as any)
|
||||
const AppOrganizationIdProjectIdLogsRoute =
|
||||
AppOrganizationIdProjectIdLogsRouteImport.update({
|
||||
id: '/logs',
|
||||
path: '/logs',
|
||||
getParentRoute: () => AppOrganizationIdProjectIdRoute,
|
||||
} as any)
|
||||
const AppOrganizationIdProjectIdInsightsRoute =
|
||||
AppOrganizationIdProjectIdInsightsRouteImport.update({
|
||||
id: '/insights',
|
||||
@@ -660,6 +667,7 @@ export interface FileRoutesByFullPath {
|
||||
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -738,6 +746,7 @@ export interface FileRoutesByTo {
|
||||
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -814,6 +823,7 @@ export interface FileRoutesById {
|
||||
'/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/_app/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -905,6 +915,7 @@ export interface FileRouteTypes {
|
||||
| '/$organizationId/$projectId/dashboards'
|
||||
| '/$organizationId/$projectId/groups'
|
||||
| '/$organizationId/$projectId/insights'
|
||||
| '/$organizationId/$projectId/logs'
|
||||
| '/$organizationId/$projectId/pages'
|
||||
| '/$organizationId/$projectId/realtime'
|
||||
| '/$organizationId/$projectId/references'
|
||||
@@ -983,6 +994,7 @@ export interface FileRouteTypes {
|
||||
| '/$organizationId/$projectId/dashboards'
|
||||
| '/$organizationId/$projectId/groups'
|
||||
| '/$organizationId/$projectId/insights'
|
||||
| '/$organizationId/$projectId/logs'
|
||||
| '/$organizationId/$projectId/pages'
|
||||
| '/$organizationId/$projectId/realtime'
|
||||
| '/$organizationId/$projectId/references'
|
||||
@@ -1058,6 +1070,7 @@ export interface FileRouteTypes {
|
||||
| '/_app/$organizationId/$projectId/dashboards'
|
||||
| '/_app/$organizationId/$projectId/groups'
|
||||
| '/_app/$organizationId/$projectId/insights'
|
||||
| '/_app/$organizationId/$projectId/logs'
|
||||
| '/_app/$organizationId/$projectId/pages'
|
||||
| '/_app/$organizationId/$projectId/realtime'
|
||||
| '/_app/$organizationId/$projectId/references'
|
||||
@@ -1444,6 +1457,13 @@ declare module '@tanstack/react-router' {
|
||||
preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport
|
||||
parentRoute: typeof AppOrganizationIdProjectIdRoute
|
||||
}
|
||||
'/_app/$organizationId/$projectId/logs': {
|
||||
id: '/_app/$organizationId/$projectId/logs'
|
||||
path: '/logs'
|
||||
fullPath: '/$organizationId/$projectId/logs'
|
||||
preLoaderRoute: typeof AppOrganizationIdProjectIdLogsRouteImport
|
||||
parentRoute: typeof AppOrganizationIdProjectIdRoute
|
||||
}
|
||||
'/_app/$organizationId/$projectId/insights': {
|
||||
id: '/_app/$organizationId/$projectId/insights'
|
||||
path: '/insights'
|
||||
@@ -2028,6 +2048,7 @@ interface AppOrganizationIdProjectIdRouteChildren {
|
||||
AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
AppOrganizationIdProjectIdLogsRoute: typeof AppOrganizationIdProjectIdLogsRoute
|
||||
AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute
|
||||
AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -2054,6 +2075,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh
|
||||
AppOrganizationIdProjectIdGroupsRoute,
|
||||
AppOrganizationIdProjectIdInsightsRoute:
|
||||
AppOrganizationIdProjectIdInsightsRoute,
|
||||
AppOrganizationIdProjectIdLogsRoute: AppOrganizationIdProjectIdLogsRoute,
|
||||
AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute,
|
||||
AppOrganizationIdProjectIdRealtimeRoute:
|
||||
AppOrganizationIdProjectIdRealtimeRoute,
|
||||
|
||||
385
apps/start/src/routes/_app.$organizationId.$projectId.logs.tsx
Normal file
385
apps/start/src/routes/_app.$organizationId.$projectId.logs.tsx
Normal file
@@ -0,0 +1,385 @@
|
||||
import { useInfiniteQuery, useQuery } from '@tanstack/react-query';
|
||||
import { createFileRoute } from '@tanstack/react-router';
|
||||
import { format } from 'date-fns';
|
||||
import { AnimatePresence, motion } from 'framer-motion';
|
||||
import {
|
||||
AlertCircleIcon,
|
||||
AlertTriangleIcon,
|
||||
BugIcon,
|
||||
ChevronDownIcon,
|
||||
ChevronRightIcon,
|
||||
InfoIcon,
|
||||
ScrollTextIcon,
|
||||
SearchIcon,
|
||||
SkullIcon,
|
||||
TerminalIcon,
|
||||
XIcon,
|
||||
} from 'lucide-react';
|
||||
import { useMemo, useState } from 'react';
|
||||
import { PageContainer } from '@/components/page-container';
|
||||
import { PageHeader } from '@/components/page-header';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { useSearchQueryState } from '@/hooks/use-search-query-state';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
|
||||
import type { ISeverityText } from '@openpanel/validation';
|
||||
|
||||
export const Route = createFileRoute(
|
||||
'/_app/$organizationId/$projectId/logs'
|
||||
)({
|
||||
component: Component,
|
||||
head: () => {
|
||||
return {
|
||||
meta: [
|
||||
{
|
||||
title: createProjectTitle(PAGE_TITLES.LOGS),
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const SEVERITY_ICONS: Record<ISeverityText, typeof InfoIcon> = {
|
||||
trace: TerminalIcon,
|
||||
debug: BugIcon,
|
||||
info: InfoIcon,
|
||||
warn: AlertTriangleIcon,
|
||||
warning: AlertTriangleIcon,
|
||||
error: AlertCircleIcon,
|
||||
fatal: SkullIcon,
|
||||
critical: SkullIcon,
|
||||
};
|
||||
|
||||
const SEVERITY_COLORS: Record<ISeverityText, string> = {
|
||||
trace: 'text-gray-400',
|
||||
debug: 'text-blue-400',
|
||||
info: 'text-green-400',
|
||||
warn: 'text-yellow-400',
|
||||
warning: 'text-yellow-400',
|
||||
error: 'text-red-400',
|
||||
fatal: 'text-red-600',
|
||||
critical: 'text-red-600',
|
||||
};
|
||||
|
||||
const SEVERITY_BG_COLORS: Record<ISeverityText, string> = {
|
||||
trace: 'bg-gray-500/10 hover:bg-gray-500/20',
|
||||
debug: 'bg-blue-500/10 hover:bg-blue-500/20',
|
||||
info: 'bg-green-500/10 hover:bg-green-500/20',
|
||||
warn: 'bg-yellow-500/10 hover:bg-yellow-500/20',
|
||||
warning: 'bg-yellow-500/10 hover:bg-yellow-500/20',
|
||||
error: 'bg-red-500/10 hover:bg-red-500/20',
|
||||
fatal: 'bg-red-600/10 hover:bg-red-600/20',
|
||||
critical: 'bg-red-600/10 hover:bg-red-600/20',
|
||||
};
|
||||
|
||||
function Component() {
|
||||
const { projectId } = Route.useParams();
|
||||
const trpc = useTRPC();
|
||||
const { search, setSearch, debouncedSearch } = useSearchQueryState();
|
||||
const [selectedSeverity, setSelectedSeverity] = useState<ISeverityText[]>([]);
|
||||
const [expandedLog, setExpandedLog] = useState<string | null>(null);
|
||||
|
||||
const severityCountsQuery = useQuery(
|
||||
trpc.log.severityCounts.queryOptions({ projectId }, { enabled: !!projectId })
|
||||
);
|
||||
|
||||
const logsQuery = useInfiniteQuery(
|
||||
trpc.log.list.infiniteQueryOptions(
|
||||
{
|
||||
projectId,
|
||||
take: 50,
|
||||
search: debouncedSearch || undefined,
|
||||
severity: selectedSeverity.length > 0 ? selectedSeverity : undefined,
|
||||
},
|
||||
{
|
||||
getNextPageParam: (lastPage) => lastPage.meta.next,
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
const logs = useMemo(() => {
|
||||
return logsQuery.data?.pages.flatMap((page) => page.data) ?? [];
|
||||
}, [logsQuery.data]);
|
||||
|
||||
const severityCounts = severityCountsQuery.data ?? {};
|
||||
const severityOptions: ISeverityText[] = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
|
||||
|
||||
const toggleSeverity = (severity: ISeverityText) => {
|
||||
setSelectedSeverity((prev) =>
|
||||
prev.includes(severity)
|
||||
? prev.filter((s) => s !== severity)
|
||||
: [...prev, severity]
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<PageContainer>
|
||||
<PageHeader
|
||||
className="mb-8"
|
||||
description="View and search device and application logs"
|
||||
icon={ScrollTextIcon}
|
||||
title="Logs"
|
||||
/>
|
||||
|
||||
{/* Severity Filter Chips */}
|
||||
<div className="mb-6 flex flex-wrap items-center gap-2">
|
||||
{severityOptions.map((severity) => {
|
||||
const count = severityCounts[severity] ?? 0;
|
||||
const isSelected = selectedSeverity.includes(severity);
|
||||
const Icon = SEVERITY_ICONS[severity];
|
||||
|
||||
return (
|
||||
<Button
|
||||
key={severity}
|
||||
className={`gap-2 capitalize ${
|
||||
isSelected ? 'ring-2 ring-primary ring-offset-2' : ''
|
||||
}`}
|
||||
onClick={() => toggleSeverity(severity)}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
>
|
||||
<Icon className={`h-4 w-4 ${SEVERITY_COLORS[severity]}`} />
|
||||
<span className="capitalize">{severity}</span>
|
||||
{count > 0 && (
|
||||
<Badge className="ml-1" variant="secondary">
|
||||
{count.toLocaleString()}
|
||||
</Badge>
|
||||
)}
|
||||
</Button>
|
||||
);
|
||||
})}
|
||||
|
||||
{selectedSeverity.length > 0 && (
|
||||
<Button
|
||||
className="gap-2"
|
||||
onClick={() => setSelectedSeverity([])}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
>
|
||||
<XIcon className="h-4 w-4" />
|
||||
Clear filters
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Search */}
|
||||
<div className="relative mb-6">
|
||||
<SearchIcon className="absolute top-1/2 left-3 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
||||
<Input
|
||||
className="pl-10"
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
placeholder="Search logs..."
|
||||
value={search}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Logs List */}
|
||||
<div className="space-y-2">
|
||||
{logs.map((log) => {
|
||||
const isExpanded = expandedLog === log.id;
|
||||
const Icon = SEVERITY_ICONS[log.severityText as ISeverityText] ?? InfoIcon;
|
||||
const severityColor = SEVERITY_COLORS[log.severityText as ISeverityText] ?? 'text-gray-400';
|
||||
const bgColor = SEVERITY_BG_COLORS[log.severityText as ISeverityText] ?? 'bg-gray-500/10';
|
||||
|
||||
return (
|
||||
<motion.div
|
||||
key={log.id}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
className={`rounded-lg border ${bgColor} transition-colors`}
|
||||
initial={{ opacity: 0, y: 10 }}
|
||||
layout
|
||||
>
|
||||
<button
|
||||
className="flex w-full items-start gap-3 p-4 text-left"
|
||||
onClick={() => setExpandedLog(isExpanded ? null : log.id)}
|
||||
type="button"
|
||||
>
|
||||
{isExpanded ? (
|
||||
<ChevronDownIcon className="mt-1 h-4 w-4 shrink-0 text-muted-foreground" />
|
||||
) : (
|
||||
<ChevronRightIcon className="mt-1 h-4 w-4 shrink-0 text-muted-foreground" />
|
||||
)}
|
||||
|
||||
<Icon className={`mt-1 h-4 w-4 shrink-0 ${severityColor}`} />
|
||||
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className={`font-mono text-xs uppercase ${severityColor}`}>
|
||||
{log.severityText}
|
||||
</span>
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{format(new Date(log.timestamp), 'MMM d, HH:mm:ss.SSS')}
|
||||
</span>
|
||||
{log.loggerName && (
|
||||
<Badge className="text-xs" variant="outline">
|
||||
{log.loggerName}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<p className="mt-1 truncate font-mono text-sm">{log.body}</p>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
<AnimatePresence>
|
||||
{isExpanded && (
|
||||
<motion.div
|
||||
animate={{ height: 'auto', opacity: 1 }}
|
||||
className="border-t px-4 pb-4"
|
||||
exit={{ height: 0, opacity: 0 }}
|
||||
initial={{ height: 0, opacity: 0 }}
|
||||
>
|
||||
<div className="space-y-4 pt-4">
|
||||
{/* Full Message */}
|
||||
<div>
|
||||
<h4 className="mb-2 font-medium text-sm">Message</h4>
|
||||
<pre className="max-h-40 overflow-auto whitespace-pre-wrap rounded bg-muted p-3 font-mono text-sm">
|
||||
{log.body}
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
{/* Attributes */}
|
||||
{Object.keys(log.attributes).length > 0 && (
|
||||
<div>
|
||||
<h4 className="mb-2 font-medium text-sm">Attributes</h4>
|
||||
<div className="grid gap-2">
|
||||
{Object.entries(log.attributes).map(([key, value]) => (
|
||||
<div
|
||||
key={key}
|
||||
className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2"
|
||||
>
|
||||
<span className="font-mono text-muted-foreground text-sm">
|
||||
{key}
|
||||
</span>
|
||||
<span className="font-mono text-sm">{value}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Resource */}
|
||||
{Object.keys(log.resource).length > 0 && (
|
||||
<div>
|
||||
<h4 className="mb-2 font-medium text-sm">Resource</h4>
|
||||
<div className="grid gap-2">
|
||||
{Object.entries(log.resource).map(([key, value]) => (
|
||||
<div
|
||||
key={key}
|
||||
className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2"
|
||||
>
|
||||
<span className="font-mono text-muted-foreground text-sm">
|
||||
{key}
|
||||
</span>
|
||||
<span className="font-mono text-sm">{value}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Trace Context */}
|
||||
{(log.traceId || log.spanId) && (
|
||||
<div>
|
||||
<h4 className="mb-2 font-medium text-sm">Trace Context</h4>
|
||||
<div className="space-y-2">
|
||||
{log.traceId && (
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="font-mono text-muted-foreground text-sm">
|
||||
Trace ID
|
||||
</span>
|
||||
<span className="font-mono text-sm">{log.traceId}</span>
|
||||
</div>
|
||||
)}
|
||||
{log.spanId && (
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="font-mono text-muted-foreground text-sm">
|
||||
Span ID
|
||||
</span>
|
||||
<span className="font-mono text-sm">{log.spanId}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Device Info */}
|
||||
<div>
|
||||
<h4 className="mb-2 font-medium text-sm">Device</h4>
|
||||
<div className="grid gap-2 text-sm">
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="text-muted-foreground">Device ID</span>
|
||||
<span className="font-mono">{log.deviceId}</span>
|
||||
</div>
|
||||
{log.profileId && (
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="text-muted-foreground">Profile ID</span>
|
||||
<span className="font-mono">{log.profileId}</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="text-muted-foreground">OS</span>
|
||||
<span>
|
||||
{log.os} {log.osVersion}
|
||||
</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="text-muted-foreground">Browser</span>
|
||||
<span>
|
||||
{log.browser} {log.browserVersion}
|
||||
</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="text-muted-foreground">Location</span>
|
||||
<span>
|
||||
{log.city}, {log.region}, {log.country}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* SDK Info */}
|
||||
<div>
|
||||
<h4 className="mb-2 font-medium text-sm">SDK</h4>
|
||||
<div className="grid gap-2 text-sm">
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="text-muted-foreground">Name</span>
|
||||
<span>{log.sdkName || 'unknown'}</span>
|
||||
</div>
|
||||
<div className="grid grid-cols-[1fr,2fr] gap-4 rounded bg-muted p-2">
|
||||
<span className="text-muted-foreground">Version</span>
|
||||
<span>{log.sdkVersion || 'unknown'}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Observed At */}
|
||||
<div className="text-muted-foreground text-xs">
|
||||
Observed at: {format(new Date(log.observedAt), 'MMM d, HH:mm:ss.SSS')}
|
||||
</div>
|
||||
</div>
|
||||
</motion.div>
|
||||
)}
|
||||
</AnimatePresence>
|
||||
</motion.div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Load More */}
|
||||
{logsQuery.hasNextPage && (
|
||||
<div className="mt-6 flex justify-center">
|
||||
<Button
|
||||
disabled={logsQuery.isFetchingNextPage}
|
||||
onClick={() => logsQuery.fetchNextPage()}
|
||||
variant="outline"
|
||||
>
|
||||
{logsQuery.isFetchingNextPage ? 'Loading...' : 'Load more'}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</PageContainer>
|
||||
);
|
||||
}
|
||||
@@ -97,6 +97,8 @@ export const PAGE_TITLES = {
|
||||
PROFILE_DETAILS: 'Profile details',
|
||||
// Groups
|
||||
GROUPS: 'Groups',
|
||||
// Logs
|
||||
LOGS: 'Logs',
|
||||
GROUP_DETAILS: 'Group details',
|
||||
|
||||
// Sub-sections
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"@openpanel/payments": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"bullmq": "^5.63.0",
|
||||
"date-fns": "^3.3.1",
|
||||
"express": "^4.18.2",
|
||||
|
||||
@@ -72,6 +72,11 @@ export async function bootCron() {
|
||||
type: 'flushGroups',
|
||||
pattern: 1000 * 10,
|
||||
},
|
||||
{
|
||||
name: 'flush',
|
||||
type: 'flushLogs',
|
||||
pattern: 1000 * 10,
|
||||
},
|
||||
{
|
||||
name: 'insightsDaily',
|
||||
type: 'insightsDaily',
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
gscQueue,
|
||||
importQueue,
|
||||
insightsQueue,
|
||||
logsQueue,
|
||||
miscQueue,
|
||||
notificationQueue,
|
||||
queueLogger,
|
||||
@@ -22,6 +23,7 @@ import { incomingEvent } from './jobs/events.incoming-event';
|
||||
import { gscJob } from './jobs/gsc';
|
||||
import { importJob } from './jobs/import';
|
||||
import { insightsProjectJob } from './jobs/insights';
|
||||
import { incomingLog } from './jobs/logs.incoming-log';
|
||||
import { miscJob } from './jobs/misc';
|
||||
import { notificationJob } from './jobs/notification';
|
||||
import { sessionsJob } from './jobs/sessions';
|
||||
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
|
||||
'import',
|
||||
'insights',
|
||||
'gsc',
|
||||
'logs',
|
||||
];
|
||||
}
|
||||
|
||||
@@ -221,6 +224,22 @@ export function bootWorkers() {
|
||||
logger.info('Started worker for gsc', { concurrency });
|
||||
}
|
||||
|
||||
// Start logs worker
|
||||
if (enabledQueues.includes('logs')) {
|
||||
const concurrency = getConcurrencyFor('logs', 10);
|
||||
const logsWorker = new Worker(logsQueue.name, async (job) => {
|
||||
const { type, payload } = job.data;
|
||||
if (type === 'incomingLog') {
|
||||
return await incomingLog(payload);
|
||||
}
|
||||
}, {
|
||||
...workerOptions,
|
||||
concurrency,
|
||||
});
|
||||
workers.push(logsWorker);
|
||||
logger.info('Started worker for logs', { concurrency });
|
||||
}
|
||||
|
||||
if (workers.length === 0) {
|
||||
logger.warn(
|
||||
'No workers started. Check ENABLED_QUEUES environment variable.'
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
eventBuffer,
|
||||
groupBuffer,
|
||||
logBuffer,
|
||||
profileBackfillBuffer,
|
||||
profileBuffer,
|
||||
replayBuffer,
|
||||
@@ -38,6 +39,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
|
||||
case 'flushGroups': {
|
||||
return await groupBuffer.tryFlush();
|
||||
}
|
||||
case 'flushLogs': {
|
||||
return await logBuffer.tryFlush();
|
||||
}
|
||||
case 'ping': {
|
||||
return await ping();
|
||||
}
|
||||
|
||||
63
apps/worker/src/jobs/logs.incoming-log.ts
Normal file
63
apps/worker/src/jobs/logs.incoming-log.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import type { IClickhouseLog } from '@openpanel/db';
|
||||
import { logBuffer } from '@openpanel/db';
|
||||
import type { LogsQueuePayload } from '@openpanel/queue';
|
||||
import { SEVERITY_TEXT_TO_NUMBER } from '@openpanel/validation';
|
||||
import { logger as baseLogger } from '@/utils/logger';
|
||||
|
||||
export async function incomingLog(
|
||||
payload: LogsQueuePayload['payload'],
|
||||
): Promise<void> {
|
||||
const logger = baseLogger.child({ projectId: payload.projectId });
|
||||
|
||||
try {
|
||||
const { log, uaInfo, geo, deviceId, sessionId, projectId, headers } = payload;
|
||||
|
||||
const sdkName = headers['openpanel-sdk-name'] ?? '';
|
||||
const sdkVersion = headers['openpanel-sdk-version'] ?? '';
|
||||
|
||||
const severityNumber =
|
||||
log.severityNumber ??
|
||||
SEVERITY_TEXT_TO_NUMBER[log.severity] ??
|
||||
9; // INFO fallback
|
||||
|
||||
const row: IClickhouseLog = {
|
||||
project_id: projectId,
|
||||
device_id: deviceId,
|
||||
profile_id: log.profileId ? String(log.profileId) : '',
|
||||
session_id: sessionId,
|
||||
timestamp: log.timestamp,
|
||||
observed_at: new Date().toISOString(),
|
||||
severity_number: severityNumber,
|
||||
severity_text: log.severity,
|
||||
body: log.body,
|
||||
trace_id: log.traceId ?? '',
|
||||
span_id: log.spanId ?? '',
|
||||
trace_flags: log.traceFlags ?? 0,
|
||||
logger_name: log.loggerName ?? '',
|
||||
attributes: log.attributes ?? {},
|
||||
resource: log.resource ?? {},
|
||||
sdk_name: sdkName,
|
||||
sdk_version: sdkVersion,
|
||||
country: geo.country ?? '',
|
||||
city: geo.city ?? '',
|
||||
region: geo.region ?? '',
|
||||
os: uaInfo.os ?? '',
|
||||
os_version: uaInfo.osVersion ?? '',
|
||||
browser: uaInfo.isServer ? '' : (uaInfo.browser ?? ''),
|
||||
browser_version: uaInfo.isServer ? '' : (uaInfo.browserVersion ?? ''),
|
||||
device: uaInfo.device ?? '',
|
||||
brand: uaInfo.isServer ? '' : (uaInfo.brand ?? ''),
|
||||
model: uaInfo.isServer ? '' : (uaInfo.model ?? ''),
|
||||
};
|
||||
|
||||
logBuffer.add(row);
|
||||
|
||||
logger.info('Log queued', {
|
||||
severity: log.severity,
|
||||
loggerName: log.loggerName,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to process incoming log', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user