3 Commits

Author SHA1 Message Date
be64494aa9 feat: add Gitea CI workflows, production compose, and update all deps
Some checks failed
Build and Push API / build-api (push) Failing after 39m34s
Build and Push Dashboard / build-dashboard (push) Has been cancelled
Build and Push Worker / build-worker (push) Has been cancelled
- Add .gitea/workflows for building and pushing api, dashboard, and worker images to git.zias.be registry
- Add docker-compose.prod.yml using pre-built registry images (no build-from-source on server)
- Update docker-compose.yml infra images to latest (postgres 18.3, redis 8.6.2, clickhouse 26.3.2.3)
- Update self-hosting/docker-compose.template.yml image versions to match
- Bump Node.js to 22.22.2 in all three Dockerfiles
- Update pnpm to 10.33.0 and upgrade all safe npm dependencies
- Add buffer-equal-constant-time patch

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-31 09:18:55 +02:00
eefbeac7f8 feat: add logs UI — tRPC router, page, and sidebar nav
- logRouter with list (infinite query, cursor-based) and severityCounts procedures
- /logs route page: severity filter chips with counts, search input, expandable log rows
- Log detail expansion shows attributes, resource, trace context, device/location
- Sidebar "Logs" link with ScrollText icon between Events and Sessions

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-30 12:08:39 +02:00
0672857974 feat: add OpenTelemetry device log capture pipeline
- ClickHouse `logs` table (migration 13) with OTel columns, bloom filter indices
- Zod validation schema for log payloads (severity, body, attributes, trace context)
- Redis-backed LogBuffer with micro-batching into ClickHouse
- POST /logs API endpoint with client auth, geo + UA enrichment
- BullMQ logs queue + worker job
- cron flushLogs every 10s wired into existing cron system
- SDK captureLog(severity, body, properties) with client-side batching

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-30 12:04:04 +02:00
65 changed files with 14586 additions and 16217 deletions

View File

@@ -0,0 +1,55 @@
name: Build and Push API
on:
push:
branches: [main]
tags: ["v*"]
pull_request:
branches: [main]
env:
REGISTRY: git.zias.be
OWNER: zias
jobs:
build-api:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ env.OWNER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=sha,prefix=sha-,format=short
type=semver,pattern={{version}}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
file: apps/api/Dockerfile
target: runner
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api:buildcache
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-api:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
build-args: |
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres

View File

@@ -0,0 +1,53 @@
name: Build and Push Dashboard
on:
push:
branches: [main]
tags: ["v*"]
pull_request:
branches: [main]
env:
REGISTRY: git.zias.be
OWNER: zias
jobs:
build-dashboard:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ env.OWNER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=sha,prefix=sha-,format=short
type=semver,pattern={{version}}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
file: apps/start/Dockerfile
target: runner
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard:buildcache
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-dashboard:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}

View File

@@ -0,0 +1,55 @@
name: Build and Push Worker
on:
push:
branches: [main]
tags: ["v*"]
pull_request:
branches: [main]
env:
REGISTRY: git.zias.be
OWNER: zias
jobs:
build-worker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ env.OWNER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=sha,prefix=sha-,format=short
type=semver,pattern={{version}}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
file: apps/worker/Dockerfile
target: runner
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker:buildcache
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-worker:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
build-args: |
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres

View File

@@ -10,14 +10,14 @@
"dependencies": { "dependencies": {
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"chalk": "^5.3.0", "chalk": "^5.6.2",
"fuzzy": "^0.1.3", "fuzzy": "^0.1.3",
"inquirer": "^9.3.5", "inquirer": "^9.3.8",
"inquirer-autocomplete-prompt": "^3.0.1", "inquirer-autocomplete-prompt": "^3.0.1",
"jiti": "^2.4.2" "jiti": "^2.6.1"
}, },
"devDependencies": { "devDependencies": {
"@types/inquirer": "^9.0.7", "@types/inquirer": "^9.0.9",
"@types/inquirer-autocomplete-prompt": "^3.0.3", "@types/inquirer-autocomplete-prompt": "^3.0.3",
"@types/node": "catalog:", "@types/node": "catalog:",
"typescript": "catalog:" "typescript": "catalog:"

View File

@@ -1,4 +1,4 @@
ARG NODE_VERSION=22.20.0 ARG NODE_VERSION=22.22.2
FROM node:${NODE_VERSION}-slim AS base FROM node:${NODE_VERSION}-slim AS base

View File

@@ -12,11 +12,11 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@ai-sdk/anthropic": "^1.2.10", "@ai-sdk/anthropic": "^1.2.12",
"@ai-sdk/openai": "^1.3.12", "@ai-sdk/openai": "^1.3.24",
"@fastify/compress": "^8.1.0", "@fastify/compress": "^8.3.1",
"@fastify/cookie": "^11.0.2", "@fastify/cookie": "^11.0.2",
"@fastify/cors": "^11.1.0", "@fastify/cors": "^11.2.0",
"@fastify/rate-limit": "^10.3.0", "@fastify/rate-limit": "^10.3.0",
"@fastify/websocket": "^11.2.0", "@fastify/websocket": "^11.2.0",
"@node-rs/argon2": "^2.0.2", "@node-rs/argon2": "^2.0.2",
@@ -33,36 +33,36 @@
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"@openpanel/trpc": "workspace:*", "@openpanel/trpc": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@trpc/server": "^11.6.0", "@trpc/server": "^11.16.0",
"ai": "^4.2.10", "ai": "^4.3.19",
"fast-json-stable-hash": "^1.0.3", "fast-json-stable-hash": "^1.0.3",
"fastify": "^5.6.1", "fastify": "^5.8.4",
"fastify-metrics": "^12.1.0", "fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0", "fastify-raw-body": "^5.0.0",
"groupmq": "catalog:", "groupmq": "catalog:",
"jsonwebtoken": "^9.0.2", "jsonwebtoken": "^9.0.3",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"sharp": "^0.33.5", "sharp": "^0.34.5",
"source-map-support": "^0.5.21", "source-map-support": "^0.5.21",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"svix": "^1.24.0", "svix": "^1.89.0",
"url-metadata": "^5.4.1", "url-metadata": "^5.4.3",
"uuid": "^9.0.1", "uuid": "^9.0.1",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@faker-js/faker": "^9.0.1", "@faker-js/faker": "^9.9.0",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/js-yaml": "^4.0.9", "@types/js-yaml": "^4.0.9",
"@types/jsonwebtoken": "^9.0.9", "@types/jsonwebtoken": "^9.0.10",
"@types/ramda": "^0.30.2", "@types/ramda": "^0.31.1",
"@types/source-map-support": "^0.5.10", "@types/source-map-support": "^0.5.10",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"@types/uuid": "^10.0.0", "@types/uuid": "^11.0.0",
"@types/ws": "^8.5.14", "@types/ws": "^8.18.1",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.1",
"tsdown": "0.14.2", "tsdown": "0.21.7",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -0,0 +1,68 @@
import { parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { getGeoLocation } from '@openpanel/geo';
import { type LogsQueuePayload, logsQueue } from '@openpanel/queue';
import { type ILogBatchPayload, zLogBatchPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { getDeviceId } from '@/utils/ids';
import { getStringHeaders } from './track.controller';
export async function handler(
request: FastifyRequest<{ Body: ILogBatchPayload }>,
reply: FastifyReply,
) {
const projectId = request.client?.projectId;
if (!projectId) {
return reply.status(400).send({ status: 400, error: 'Missing projectId' });
}
const validationResult = zLogBatchPayload.safeParse(request.body);
if (!validationResult.success) {
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Validation failed',
errors: validationResult.error.errors,
});
}
const { logs } = validationResult.data;
const ip = request.clientIp;
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
const headers = getStringHeaders(request.headers);
const receivedAt = new Date().toISOString();
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
const { deviceId, sessionId } = await getDeviceId({ projectId, ip, ua, salts });
const uaInfo = parseUserAgent(ua, undefined);
const jobs: LogsQueuePayload[] = logs.map((log) => ({
type: 'incomingLog' as const,
payload: {
projectId,
log: {
...log,
timestamp: log.timestamp ?? receivedAt,
},
uaInfo,
geo: {
country: geo.country,
city: geo.city,
region: geo.region,
},
headers,
deviceId,
sessionId,
},
}));
await logsQueue.addBulk(
jobs.map((job) => ({
name: 'incomingLog',
data: job,
})),
);
return reply.status(200).send({ ok: true, count: logs.length });
}

View File

@@ -44,6 +44,7 @@ import manageRouter from './routes/manage.router';
import miscRouter from './routes/misc.router'; import miscRouter from './routes/misc.router';
import oauthRouter from './routes/oauth-callback.router'; import oauthRouter from './routes/oauth-callback.router';
import profileRouter from './routes/profile.router'; import profileRouter from './routes/profile.router';
import logsRouter from './routes/logs.router';
import trackRouter from './routes/track.router'; import trackRouter from './routes/track.router';
import webhookRouter from './routes/webhook.router'; import webhookRouter from './routes/webhook.router';
import { HttpError } from './utils/errors'; import { HttpError } from './utils/errors';
@@ -209,6 +210,7 @@ const startServer = async () => {
instance.register(importRouter, { prefix: '/import' }); instance.register(importRouter, { prefix: '/import' });
instance.register(insightsRouter, { prefix: '/insights' }); instance.register(insightsRouter, { prefix: '/insights' });
instance.register(trackRouter, { prefix: '/track' }); instance.register(trackRouter, { prefix: '/track' });
instance.register(logsRouter, { prefix: '/logs' });
instance.register(manageRouter, { prefix: '/manage' }); instance.register(manageRouter, { prefix: '/manage' });
// Keep existing endpoints for backward compatibility // Keep existing endpoints for backward compatibility
instance.get('/healthcheck', healthcheck); instance.get('/healthcheck', healthcheck);

View File

@@ -0,0 +1,17 @@
import type { FastifyPluginCallback } from 'fastify';
import { handler } from '@/controllers/logs.controller';
import { clientHook } from '@/hooks/client.hook';
import { duplicateHook } from '@/hooks/duplicate.hook';
const logsRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preValidation', duplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.route({
method: 'POST',
url: '/',
handler,
});
};
export default logsRouter;

View File

@@ -16,11 +16,11 @@
}, },
"dependencies": { "dependencies": {
"@nivo/funnel": "^0.99.0", "@nivo/funnel": "^0.99.0",
"@number-flow/react": "0.5.10", "@number-flow/react": "0.6.0",
"@opennextjs/cloudflare": "^1.17.1", "@opennextjs/cloudflare": "^1.18.0",
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/geo": "workspace:*", "@openpanel/geo": "workspace:*",
"@openpanel/nextjs": "^1.2.0", "@openpanel/nextjs": "^1.4.0",
"@openpanel/payments": "workspace:^", "@openpanel/payments": "workspace:^",
"@openpanel/sdk-info": "workspace:^", "@openpanel/sdk-info": "workspace:^",
"@openstatus/react": "0.0.3", "@openstatus/react": "0.0.3",
@@ -28,37 +28,37 @@
"@radix-ui/react-slider": "1.3.6", "@radix-ui/react-slider": "1.3.6",
"@radix-ui/react-slot": "1.2.4", "@radix-ui/react-slot": "1.2.4",
"@radix-ui/react-tooltip": "1.2.8", "@radix-ui/react-tooltip": "1.2.8",
"cheerio": "^1.0.0", "cheerio": "^1.2.0",
"class-variance-authority": "0.7.1", "class-variance-authority": "0.7.1",
"clsx": "2.1.1", "clsx": "2.1.1",
"dotted-map": "2.2.3", "dotted-map": "3.1.0",
"framer-motion": "12.23.25", "framer-motion": "12.38.0",
"fumadocs-core": "16.2.2", "fumadocs-core": "16.7.7",
"fumadocs-mdx": "14.0.4", "fumadocs-mdx": "14.2.11",
"fumadocs-ui": "16.2.2", "fumadocs-ui": "16.7.7",
"geist": "1.5.1", "geist": "1.7.0",
"lucide-react": "^0.555.0", "lucide-react": "^0.555.0",
"next": "16.0.7", "next": "16.2.1",
"next-themes": "^0.4.6", "next-themes": "^0.4.6",
"react": "catalog:", "react": "catalog:",
"react-dom": "catalog:", "react-dom": "catalog:",
"react-markdown": "^10.1.0", "react-markdown": "^10.1.0",
"recharts": "^2.15.0", "recharts": "^2.15.4",
"rehype-external-links": "3.0.0", "rehype-external-links": "3.0.0",
"tailwind-merge": "3.4.0", "tailwind-merge": "3.5.0",
"tailwindcss-animate": "1.0.7", "tailwindcss-animate": "1.0.7",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@tailwindcss/postcss": "^4.1.17", "@tailwindcss/postcss": "^4.2.2",
"@types/mdx": "^2.0.13", "@types/mdx": "^2.0.13",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/react": "catalog:", "@types/react": "catalog:",
"@types/react-dom": "catalog:", "@types/react-dom": "catalog:",
"autoprefixer": "^10.4.22", "autoprefixer": "^10.4.27",
"postcss": "^8.5.6", "postcss": "^8.5.8",
"tailwindcss": "4.1.17", "tailwindcss": "4.2.2",
"typescript": "catalog:", "typescript": "catalog:",
"wrangler": "^4.65.0" "wrangler": "^4.78.0"
} }
} }

View File

@@ -1,4 +1,4 @@
ARG NODE_VERSION=22.20.0 ARG NODE_VERSION=22.22.2
FROM node:${NODE_VERSION}-slim AS base FROM node:${NODE_VERSION}-slim AS base

View File

@@ -17,21 +17,21 @@
"with-env": "dotenv -e ../../.env -c --" "with-env": "dotenv -e ../../.env -c --"
}, },
"dependencies": { "dependencies": {
"@ai-sdk/react": "^1.2.5", "@ai-sdk/react": "^1.2.12",
"@codemirror/commands": "^6.7.0", "@codemirror/commands": "^6.10.3",
"@codemirror/lang-javascript": "^6.2.0", "@codemirror/lang-javascript": "^6.2.5",
"@codemirror/lang-json": "^6.0.1", "@codemirror/lang-json": "^6.0.2",
"@codemirror/state": "^6.4.0", "@codemirror/state": "^6.6.0",
"@codemirror/theme-one-dark": "^6.1.3", "@codemirror/theme-one-dark": "^6.1.3",
"@codemirror/view": "^6.35.0", "@codemirror/view": "^6.40.0",
"@dnd-kit/core": "^6.3.1", "@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0", "@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2", "@dnd-kit/utilities": "^3.2.2",
"@faker-js/faker": "^9.6.0", "@faker-js/faker": "^9.9.0",
"@hookform/resolvers": "^3.3.4", "@hookform/resolvers": "^3.10.0",
"@hyperdx/node-opentelemetry": "^0.8.1", "@hyperdx/node-opentelemetry": "^0.10.3",
"@nivo/sankey": "^0.99.0", "@nivo/sankey": "^0.99.0",
"@number-flow/react": "0.5.10", "@number-flow/react": "0.6.0",
"@openpanel/common": "workspace:^", "@openpanel/common": "workspace:^",
"@openpanel/constants": "workspace:^", "@openpanel/constants": "workspace:^",
"@openpanel/importer": "workspace:^", "@openpanel/importer": "workspace:^",
@@ -40,100 +40,100 @@
"@openpanel/payments": "workspace:*", "@openpanel/payments": "workspace:*",
"@openpanel/sdk-info": "workspace:^", "@openpanel/sdk-info": "workspace:^",
"@openpanel/validation": "workspace:^", "@openpanel/validation": "workspace:^",
"@openpanel/web": "^1.0.1", "@openpanel/web": "^1.3.0",
"@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-alert-dialog": "^1.1.15",
"@radix-ui/react-aspect-ratio": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.8",
"@radix-ui/react-avatar": "^1.1.10", "@radix-ui/react-avatar": "^1.1.11",
"@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-checkbox": "^1.3.3",
"@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dialog": "^1.1.15",
"@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-dropdown-menu": "^2.1.16",
"@radix-ui/react-label": "^2.1.7", "@radix-ui/react-label": "^2.1.8",
"@radix-ui/react-popover": "^1.1.15", "@radix-ui/react-popover": "^1.1.15",
"@radix-ui/react-portal": "^1.1.9", "@radix-ui/react-portal": "^1.1.10",
"@radix-ui/react-progress": "^1.1.7", "@radix-ui/react-progress": "^1.1.8",
"@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-radio-group": "^1.3.8",
"@radix-ui/react-scroll-area": "^1.2.10", "@radix-ui/react-scroll-area": "^1.2.10",
"@radix-ui/react-select": "^2.2.6", "@radix-ui/react-select": "^2.2.6",
"@radix-ui/react-separator": "^1.1.7", "@radix-ui/react-separator": "^1.1.8",
"@radix-ui/react-slider": "1.2.3", "@radix-ui/react-slider": "1.3.6",
"@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-slot": "^1.2.4",
"@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-switch": "^1.2.6",
"@radix-ui/react-tabs": "^1.1.13", "@radix-ui/react-tabs": "^1.1.13",
"@radix-ui/react-toast": "^1.2.15", "@radix-ui/react-toast": "^1.2.15",
"@radix-ui/react-toggle": "^1.1.10", "@radix-ui/react-toggle": "^1.1.10",
"@radix-ui/react-toggle-group": "^1.1.11", "@radix-ui/react-toggle-group": "^1.1.11",
"@radix-ui/react-tooltip": "^1.2.8", "@radix-ui/react-tooltip": "^1.2.8",
"@reduxjs/toolkit": "^2.8.2", "@reduxjs/toolkit": "^2.11.2",
"@sentry/tanstackstart-react": "^9.12.0", "@sentry/tanstackstart-react": "^10.46.0",
"@tailwindcss/container-queries": "^0.1.1", "@tailwindcss/container-queries": "^0.1.1",
"@tailwindcss/typography": "^0.5.15", "@tailwindcss/typography": "^0.5.19",
"@tailwindcss/vite": "^4.0.6", "@tailwindcss/vite": "^4.2.2",
"@tanstack/match-sorter-utils": "^8.19.4", "@tanstack/match-sorter-utils": "^8.19.4",
"@tanstack/nitro-v2-vite-plugin": "^1.133.19", "@tanstack/nitro-v2-vite-plugin": "^1.154.9",
"@tanstack/react-devtools": "^0.7.6", "@tanstack/react-devtools": "^0.10.0",
"@tanstack/react-query": "^5.90.2", "@tanstack/react-query": "^5.95.2",
"@tanstack/react-query-devtools": "^5.90.2", "@tanstack/react-query-devtools": "^5.95.2",
"@tanstack/react-router": "^1.132.47", "@tanstack/react-router": "^1.168.10",
"@tanstack/react-router-devtools": "^1.132.51", "@tanstack/react-router-devtools": "^1.166.11",
"@tanstack/react-router-ssr-query": "^1.132.47", "@tanstack/react-router-ssr-query": "^1.166.10",
"@tanstack/react-start": "^1.132.56", "@tanstack/react-start": "^1.167.16",
"@tanstack/react-store": "^0.8.0", "@tanstack/react-store": "^0.9.3",
"@tanstack/react-table": "^8.21.3", "@tanstack/react-table": "^8.21.3",
"@tanstack/react-virtual": "^3.13.12", "@tanstack/react-virtual": "^3.13.23",
"@tanstack/router-plugin": "^1.132.56", "@tanstack/router-plugin": "^1.167.12",
"@tanstack/store": "^0.8.0", "@tanstack/store": "^0.9.3",
"@trpc/client": "^11.6.0", "@trpc/client": "^11.16.0",
"@trpc/react-query": "^11.6.0", "@trpc/react-query": "^11.16.0",
"@trpc/server": "^11.6.0", "@trpc/server": "^11.16.0",
"@trpc/tanstack-react-query": "^11.6.0", "@trpc/tanstack-react-query": "^11.16.0",
"@types/d3": "^7.4.3", "@types/d3": "^7.4.3",
"ai": "^4.2.10", "ai": "^4.3.19",
"bind-event-listener": "^3.0.0", "bind-event-listener": "^3.0.0",
"class-variance-authority": "^0.7.1", "class-variance-authority": "^0.7.1",
"clsx": "^2.1.1", "clsx": "^2.1.1",
"cmdk": "^0.2.1", "cmdk": "^0.2.1",
"codemirror": "^6.0.1", "codemirror": "^6.0.2",
"d3": "^7.8.5", "d3": "^7.9.0",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"debounce": "^2.2.0", "debounce": "^3.0.0",
"embla-carousel-autoplay": "^8.6.0", "embla-carousel-autoplay": "^8.6.0",
"embla-carousel-react": "8.0.0-rc22", "embla-carousel-react": "8.6.0",
"flag-icons": "^7.1.0", "flag-icons": "^7.5.0",
"framer-motion": "^11.0.28", "framer-motion": "^12.38.0",
"hamburger-react": "^2.5.0", "hamburger-react": "^2.5.2",
"input-otp": "^1.2.4", "input-otp": "^1.4.2",
"javascript-time-ago": "^2.5.9", "javascript-time-ago": "^2.6.4",
"katex": "^0.16.21", "katex": "^0.16.44",
"lodash.debounce": "^4.0.8", "lodash.debounce": "^4.0.8",
"lodash.isequal": "^4.5.0", "lodash.isequal": "^4.5.0",
"lodash.throttle": "^4.1.1", "lodash.throttle": "^4.1.1",
"lottie-react": "^2.4.0", "lottie-react": "^2.4.1",
"lucide-react": "^0.476.0", "lucide-react": "^0.476.0",
"mitt": "^3.0.1", "mitt": "^3.0.1",
"nuqs": "^2.5.2", "nuqs": "^2.8.9",
"prisma-error-enum": "^0.1.3", "prisma-error-enum": "^0.1.3",
"pushmodal": "^1.0.3", "pushmodal": "^1.0.5",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"random-animal-name": "^0.1.1", "random-animal-name": "^0.1.1",
"rc-virtual-list": "^3.14.5", "rc-virtual-list": "^3.19.2",
"react": "catalog:", "react": "catalog:",
"react-animate-height": "^3.2.3", "react-animate-height": "^3.2.3",
"react-animated-numbers": "^1.1.1", "react-animated-numbers": "^1.1.1",
"react-day-picker": "^9.9.0", "react-day-picker": "^9.14.0",
"react-dom": "catalog:", "react-dom": "catalog:",
"react-grid-layout": "^1.5.2", "react-grid-layout": "^1.5.3",
"react-hook-form": "^7.50.1", "react-hook-form": "^7.72.0",
"react-in-viewport": "1.0.0-beta.8", "react-in-viewport": "1.0.0-beta.9",
"react-markdown": "^10.1.0", "react-markdown": "^10.1.0",
"react-redux": "^8.1.3", "react-redux": "^8.1.3",
"react-resizable": "^3.0.5", "react-resizable": "^3.1.3",
"react-responsive": "^9.0.2", "react-responsive": "^10.0.1",
"react-simple-maps": "3.0.0", "react-simple-maps": "3.0.0",
"react-svg-worldmap": "2.0.0-alpha.16", "react-svg-worldmap": "2.0.1",
"react-syntax-highlighter": "^15.5.0", "react-syntax-highlighter": "^16.1.1",
"react-use-websocket": "^4.7.0", "react-use-websocket": "^4.13.0",
"react-virtualized-auto-sizer": "^1.0.22", "react-virtualized-auto-sizer": "^1.0.26",
"recharts": "^2.15.4", "recharts": "^2.15.4",
"rehype-katex": "^7.0.1", "rehype-katex": "^7.0.1",
"remark-gfm": "^4.0.1", "remark-gfm": "^4.0.1",
@@ -142,42 +142,42 @@
"remark-parse": "^11.0.0", "remark-parse": "^11.0.0",
"remark-rehype": "^11.1.2", "remark-rehype": "^11.1.2",
"rrweb-player": "2.0.0-alpha.20", "rrweb-player": "2.0.0-alpha.20",
"short-unique-id": "^5.0.3", "short-unique-id": "^5.3.2",
"slugify": "^1.6.6", "slugify": "^1.6.8",
"sonner": "^1.4.0", "sonner": "^1.7.4",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^2.2.2", "superjson": "^2.2.6",
"tailwind-merge": "^3.0.2", "tailwind-merge": "^3.5.0",
"tailwindcss": "^4.0.6", "tailwindcss": "^4.2.2",
"tailwindcss-animate": "^1.0.7", "tailwindcss-animate": "^1.0.7",
"tw-animate-css": "^1.3.6", "tw-animate-css": "^1.4.0",
"usehooks-ts": "^2.14.0", "usehooks-ts": "^2.16.0",
"vite-tsconfig-paths": "^5.1.4", "vite-tsconfig-paths": "^6.1.1",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "1.9.4", "@biomejs/biome": "2.4.10",
"@cloudflare/vite-plugin": "1.20.3", "@cloudflare/vite-plugin": "1.30.2",
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/trpc": "workspace:*", "@openpanel/trpc": "workspace:*",
"@tanstack/devtools-event-client": "^0.3.3", "@tanstack/devtools-event-client": "^0.4.3",
"@testing-library/dom": "^10.4.0", "@testing-library/dom": "^10.4.1",
"@testing-library/react": "^16.2.0", "@testing-library/react": "^16.3.2",
"@types/lodash.debounce": "^4.0.9", "@types/lodash.debounce": "^4.0.9",
"@types/lodash.isequal": "^4.5.8", "@types/lodash.isequal": "^4.5.8",
"@types/lodash.throttle": "^4.1.9", "@types/lodash.throttle": "^4.1.9",
"@types/ramda": "^0.31.0", "@types/ramda": "^0.31.1",
"@types/react": "catalog:", "@types/react": "catalog:",
"@types/react-dom": "catalog:", "@types/react-dom": "catalog:",
"@types/react-grid-layout": "^1.3.5", "@types/react-grid-layout": "^2.1.0",
"@types/react-simple-maps": "^3.0.4", "@types/react-simple-maps": "^3.0.6",
"@types/react-syntax-highlighter": "^15.5.11", "@types/react-syntax-highlighter": "^15.5.13",
"@vitejs/plugin-react": "^4.3.4", "@vitejs/plugin-react": "^4.7.0",
"jsdom": "^26.0.0", "jsdom": "^26.1.0",
"typescript": "catalog:", "typescript": "catalog:",
"vite": "^6.3.5", "vite": "^6.4.1",
"vitest": "^3.0.5", "vitest": "^3.2.4",
"web-vitals": "^4.2.4", "web-vitals": "^5.2.0",
"wrangler": "4.59.1" "wrangler": "4.78.0"
} }
} }

View File

@@ -15,6 +15,7 @@ import {
LayoutDashboardIcon, LayoutDashboardIcon,
LayoutPanelTopIcon, LayoutPanelTopIcon,
PlusIcon, PlusIcon,
ScrollTextIcon,
SearchIcon, SearchIcon,
SparklesIcon, SparklesIcon,
TrendingUpDownIcon, TrendingUpDownIcon,
@@ -61,6 +62,7 @@ export default function SidebarProjectMenu({
<SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" /> <SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" />
<SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" /> <SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" />
<SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" /> <SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" />
<SidebarLink href={'/logs'} icon={ScrollTextIcon} label="Logs" />
<SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" /> <SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" />
<SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" /> <SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" />
<SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" /> <SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" />

View File

@@ -47,6 +47,7 @@ import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/
import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references' import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references'
import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime' import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime'
import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages' import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages'
import { Route as AppOrganizationIdProjectIdLogsRouteImport } from './routes/_app.$organizationId.$projectId.logs'
import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights' import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights'
import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups' import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups'
import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards' import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards'
@@ -352,6 +353,12 @@ const AppOrganizationIdProjectIdPagesRoute =
path: '/pages', path: '/pages',
getParentRoute: () => AppOrganizationIdProjectIdRoute, getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any) } as any)
const AppOrganizationIdProjectIdLogsRoute =
AppOrganizationIdProjectIdLogsRouteImport.update({
id: '/logs',
path: '/logs',
getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any)
const AppOrganizationIdProjectIdInsightsRoute = const AppOrganizationIdProjectIdInsightsRoute =
AppOrganizationIdProjectIdInsightsRouteImport.update({ AppOrganizationIdProjectIdInsightsRouteImport.update({
id: '/insights', id: '/insights',
@@ -660,6 +667,7 @@ export interface FileRoutesByFullPath {
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute '/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute '/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute '/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute '/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -738,6 +746,7 @@ export interface FileRoutesByTo {
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute '/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute '/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute '/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute '/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -814,6 +823,7 @@ export interface FileRoutesById {
'/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute '/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute '/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute '/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/_app/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute '/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -905,6 +915,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/dashboards' | '/$organizationId/$projectId/dashboards'
| '/$organizationId/$projectId/groups' | '/$organizationId/$projectId/groups'
| '/$organizationId/$projectId/insights' | '/$organizationId/$projectId/insights'
| '/$organizationId/$projectId/logs'
| '/$organizationId/$projectId/pages' | '/$organizationId/$projectId/pages'
| '/$organizationId/$projectId/realtime' | '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references' | '/$organizationId/$projectId/references'
@@ -983,6 +994,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/dashboards' | '/$organizationId/$projectId/dashboards'
| '/$organizationId/$projectId/groups' | '/$organizationId/$projectId/groups'
| '/$organizationId/$projectId/insights' | '/$organizationId/$projectId/insights'
| '/$organizationId/$projectId/logs'
| '/$organizationId/$projectId/pages' | '/$organizationId/$projectId/pages'
| '/$organizationId/$projectId/realtime' | '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references' | '/$organizationId/$projectId/references'
@@ -1058,6 +1070,7 @@ export interface FileRouteTypes {
| '/_app/$organizationId/$projectId/dashboards' | '/_app/$organizationId/$projectId/dashboards'
| '/_app/$organizationId/$projectId/groups' | '/_app/$organizationId/$projectId/groups'
| '/_app/$organizationId/$projectId/insights' | '/_app/$organizationId/$projectId/insights'
| '/_app/$organizationId/$projectId/logs'
| '/_app/$organizationId/$projectId/pages' | '/_app/$organizationId/$projectId/pages'
| '/_app/$organizationId/$projectId/realtime' | '/_app/$organizationId/$projectId/realtime'
| '/_app/$organizationId/$projectId/references' | '/_app/$organizationId/$projectId/references'
@@ -1444,6 +1457,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute parentRoute: typeof AppOrganizationIdProjectIdRoute
} }
'/_app/$organizationId/$projectId/logs': {
id: '/_app/$organizationId/$projectId/logs'
path: '/logs'
fullPath: '/$organizationId/$projectId/logs'
preLoaderRoute: typeof AppOrganizationIdProjectIdLogsRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute
}
'/_app/$organizationId/$projectId/insights': { '/_app/$organizationId/$projectId/insights': {
id: '/_app/$organizationId/$projectId/insights' id: '/_app/$organizationId/$projectId/insights'
path: '/insights' path: '/insights'
@@ -2028,6 +2048,7 @@ interface AppOrganizationIdProjectIdRouteChildren {
AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute
AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute
AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute
AppOrganizationIdProjectIdLogsRoute: typeof AppOrganizationIdProjectIdLogsRoute
AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute
AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute
AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute
@@ -2054,6 +2075,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh
AppOrganizationIdProjectIdGroupsRoute, AppOrganizationIdProjectIdGroupsRoute,
AppOrganizationIdProjectIdInsightsRoute: AppOrganizationIdProjectIdInsightsRoute:
AppOrganizationIdProjectIdInsightsRoute, AppOrganizationIdProjectIdInsightsRoute,
AppOrganizationIdProjectIdLogsRoute: AppOrganizationIdProjectIdLogsRoute,
AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute, AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute,
AppOrganizationIdProjectIdRealtimeRoute: AppOrganizationIdProjectIdRealtimeRoute:
AppOrganizationIdProjectIdRealtimeRoute, AppOrganizationIdProjectIdRealtimeRoute,

View File

@@ -0,0 +1,340 @@
import { useInfiniteQuery, useQuery } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import {
AlertCircleIcon,
AlertTriangleIcon,
BugIcon,
ChevronDownIcon,
ChevronRightIcon,
InfoIcon,
SearchIcon,
XCircleIcon,
} from 'lucide-react';
import { parseAsArrayOf, parseAsString, useQueryState } from 'nuqs';
import { useState } from 'react';
import { PageContainer } from '@/components/page-container';
import { PageHeader } from '@/components/page-header';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { useTRPC } from '@/integrations/trpc/react';
import { cn } from '@/utils/cn';
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
import type { IServiceLog } from '@openpanel/trpc';
export const Route = createFileRoute(
'/_app/$organizationId/$projectId/logs',
)({
component: Component,
head: () => ({
meta: [{ title: createProjectTitle(PAGE_TITLES.LOGS) }],
}),
});
const SEVERITY_LEVELS = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'] as const;
type SeverityLevel = (typeof SEVERITY_LEVELS)[number];
function getSeverityVariant(
severity: string,
): 'default' | 'secondary' | 'info' | 'warning' | 'destructive' | 'outline' {
switch (severity.toLowerCase()) {
case 'fatal':
case 'critical':
case 'error':
return 'destructive';
case 'warn':
case 'warning':
return 'warning';
case 'info':
return 'info';
default:
return 'outline';
}
}
function SeverityIcon({ severity }: { severity: string }) {
const s = severity.toLowerCase();
const cls = 'size-3.5 shrink-0';
if (s === 'fatal' || s === 'critical') return <XCircleIcon className={cn(cls, 'text-destructive')} />;
if (s === 'error') return <AlertCircleIcon className={cn(cls, 'text-destructive')} />;
if (s === 'warn' || s === 'warning') return <AlertTriangleIcon className={cn(cls, 'text-yellow-500')} />;
if (s === 'debug' || s === 'trace') return <BugIcon className={cn(cls, 'text-muted-foreground')} />;
return <InfoIcon className={cn(cls, 'text-blue-500')} />;
}
function LogRow({ log }: { log: IServiceLog }) {
const [expanded, setExpanded] = useState(false);
const hasDetails =
(log.attributes && Object.keys(log.attributes).length > 0) ||
(log.resource && Object.keys(log.resource).length > 0) ||
log.traceId ||
log.loggerName;
return (
<>
<button
type="button"
className={cn(
'grid w-full grid-cols-[28px_90px_80px_1fr_120px] items-center gap-3 px-4 py-2.5 text-left text-sm transition-colors hover:bg-muted/50 border-b border-border/50',
expanded && 'bg-muted/30',
)}
onClick={() => hasDetails && setExpanded((v) => !v)}
>
<span className="flex items-center justify-center text-muted-foreground">
{hasDetails ? (
expanded ? (
<ChevronDownIcon className="size-3.5" />
) : (
<ChevronRightIcon className="size-3.5" />
)
) : null}
</span>
<span className="text-muted-foreground tabular-nums text-xs">
{log.timestamp.toLocaleTimeString()}
</span>
<span>
<Badge variant={getSeverityVariant(log.severityText)} className="gap-1 font-mono uppercase text-[10px]">
<SeverityIcon severity={log.severityText} />
{log.severityText}
</Badge>
</span>
<span className="truncate font-mono text-xs">{log.body}</span>
<span className="truncate text-xs text-muted-foreground text-right">
{log.loggerName || log.os || log.device || '—'}
</span>
</button>
{expanded && (
<div className="border-b border-border/50 bg-muted/20 px-4 py-3">
<div className="grid gap-4 pl-[calc(28px+0.75rem)] text-xs sm:grid-cols-2">
{log.loggerName && (
<MetaRow label="Logger" value={log.loggerName} />
)}
{log.traceId && (
<MetaRow label="Trace ID" value={log.traceId} mono />
)}
{log.spanId && (
<MetaRow label="Span ID" value={log.spanId} mono />
)}
{log.deviceId && (
<MetaRow label="Device ID" value={log.deviceId} mono />
)}
{log.os && (
<MetaRow label="OS" value={`${log.os} ${log.osVersion}`.trim()} />
)}
{log.country && (
<MetaRow label="Location" value={[log.city, log.country].filter(Boolean).join(', ')} />
)}
{Object.keys(log.attributes).length > 0 && (
<div className="col-span-2">
<p className="mb-1.5 font-semibold text-muted-foreground">Attributes</p>
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
{Object.entries(log.attributes).map(([k, v]) => (
<div key={k} className="flex gap-2">
<span className="text-blue-500 shrink-0">{k}</span>
<span className="text-muted-foreground">=</span>
<span className="break-all">{v}</span>
</div>
))}
</div>
</div>
)}
{Object.keys(log.resource).length > 0 && (
<div className="col-span-2">
<p className="mb-1.5 font-semibold text-muted-foreground">Resource</p>
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
{Object.entries(log.resource).map(([k, v]) => (
<div key={k} className="flex gap-2">
<span className="text-emerald-600 shrink-0">{k}</span>
<span className="text-muted-foreground">=</span>
<span className="break-all">{v}</span>
</div>
))}
</div>
</div>
)}
</div>
</div>
)}
</>
);
}
function MetaRow({ label, value, mono }: { label: string; value: string; mono?: boolean }) {
return (
<div className="flex gap-2">
<span className="w-24 shrink-0 font-medium text-muted-foreground">{label}</span>
<span className={cn('break-all', mono && 'font-mono')}>{value}</span>
</div>
);
}
function Component() {
const { projectId } = Route.useParams();
const trpc = useTRPC();
const [search, setSearch] = useQueryState('search', parseAsString.withDefault(''));
const [severities, setSeverities] = useQueryState(
'severity',
parseAsArrayOf(parseAsString).withDefault([]),
);
const countsQuery = useQuery(
trpc.log.severityCounts.queryOptions({ projectId }),
);
const logsQuery = useInfiniteQuery(
trpc.log.list.infiniteQueryOptions(
{
projectId,
search: search || undefined,
severity: severities.length > 0 ? (severities as SeverityLevel[]) : undefined,
take: 50,
},
{
getNextPageParam: (lastPage) => lastPage.meta.next,
},
),
);
const logs = logsQuery.data?.pages.flatMap((p) => p.data) ?? [];
const counts = countsQuery.data ?? {};
const toggleSeverity = (s: string) => {
setSeverities((prev) =>
prev.includes(s) ? prev.filter((x) => x !== s) : [...prev, s],
);
};
return (
<PageContainer>
<PageHeader
className="mb-6"
title="Logs"
description="Captured device and application logs in OpenTelemetry format"
/>
{/* Severity filter chips */}
<div className="mb-4 flex flex-wrap items-center gap-2">
{SEVERITY_LEVELS.map((s) => {
const active = severities.includes(s);
const count = counts[s] ?? 0;
return (
<button
key={s}
type="button"
onClick={() => toggleSeverity(s)}
className={cn(
'inline-flex items-center gap-1.5 rounded-full border px-3 py-1 text-xs font-medium transition-colors',
active
? 'border-foreground bg-foreground text-background'
: 'border-border bg-transparent text-muted-foreground hover:border-foreground/50 hover:text-foreground',
)}
>
<SeverityIcon severity={s} />
<span className="uppercase">{s}</span>
{count > 0 && (
<span className={cn('tabular-nums', active ? 'opacity-70' : 'opacity-50')}>
{count.toLocaleString()}
</span>
)}
</button>
);
})}
{severities.length > 0 && (
<button
type="button"
onClick={() => setSeverities([])}
className="text-xs text-muted-foreground hover:text-foreground underline"
>
Clear
</button>
)}
</div>
{/* Search */}
<div className="mb-4 relative">
<SearchIcon className="absolute left-3 top-1/2 -translate-y-1/2 size-4 text-muted-foreground" />
<Input
className="pl-9"
placeholder="Search log messages…"
value={search}
onChange={(e) => setSearch(e.target.value || null)}
/>
</div>
{/* Log table */}
<div className="rounded-lg border bg-card overflow-hidden">
{/* Header */}
<div className="grid grid-cols-[28px_90px_80px_1fr_120px] gap-3 border-b bg-muted/50 px-4 py-2 text-xs font-medium text-muted-foreground">
<span />
<span>Time</span>
<span>Level</span>
<span>Message</span>
<span className="text-right">Source</span>
</div>
{logsQuery.isPending && (
<div className="space-y-0 divide-y">
{Array.from({ length: 10 }).map((_, i) => (
<div key={i} className="h-10 animate-pulse bg-muted/30" />
))}
</div>
)}
{!logsQuery.isPending && logs.length === 0 && (
<div className="flex flex-col items-center gap-2 py-16 text-center text-muted-foreground">
<ScrollTextIcon className="size-10 opacity-30" />
<p className="text-sm font-medium">No logs found</p>
<p className="text-xs">
{search || severities.length > 0
? 'Try adjusting your filters'
: 'Logs will appear here once your app starts sending them'}
</p>
</div>
)}
{logs.map((log) => (
<LogRow key={log.id} log={log} />
))}
{logsQuery.hasNextPage && (
<div className="flex justify-center p-4">
<Button
variant="outline"
size="sm"
onClick={() => logsQuery.fetchNextPage()}
disabled={logsQuery.isFetchingNextPage}
>
{logsQuery.isFetchingNextPage ? 'Loading…' : 'Load more'}
</Button>
</div>
)}
</div>
</PageContainer>
);
}
function ScrollTextIcon({ className }: { className?: string }) {
return (
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
className={className}
>
<path d="M8 21h12a2 2 0 0 0 2-2v-2H10v2a2 2 0 1 1-4 0V5a2 2 0 1 0-4 0v3h4" />
<path d="M19 17V5a2 2 0 0 0-2-2H4" />
<path d="M15 8h-5" />
<path d="M15 12h-5" />
</svg>
);
}

View File

@@ -88,6 +88,7 @@ export const PAGE_TITLES = {
MEMBERS: 'Members', MEMBERS: 'Members',
BILLING: 'Billing', BILLING: 'Billing',
CHAT: 'AI Assistant', CHAT: 'AI Assistant',
LOGS: 'Logs',
REALTIME: 'Realtime', REALTIME: 'Realtime',
REFERENCES: 'References', REFERENCES: 'References',
INSIGHTS: 'Insights', INSIGHTS: 'Insights',

View File

@@ -10,15 +10,15 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/web": "workspace:*", "@openpanel/web": "workspace:*",
"react": "^19.0.0", "react": "^19.2.4",
"react-dom": "^19.0.0", "react-dom": "^19.2.4",
"react-router-dom": "^7.13.1" "react-router-dom": "^7.13.2"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "^19.0.0", "@types/react": "^19.2.14",
"@types/react-dom": "^19.0.0", "@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^4.3.4", "@vitejs/plugin-react": "^4.7.0",
"typescript": "catalog:", "typescript": "catalog:",
"vite": "^6.0.0" "vite": "^6.4.1"
} }
} }

View File

@@ -1,4 +1,4 @@
ARG NODE_VERSION=22.20.0 ARG NODE_VERSION=22.22.2
FROM node:${NODE_VERSION}-slim AS base FROM node:${NODE_VERSION}-slim AS base

View File

@@ -11,8 +11,8 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@bull-board/api": "6.14.0", "@bull-board/api": "6.20.6",
"@bull-board/express": "6.14.0", "@bull-board/express": "6.20.6",
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*", "@openpanel/email": "workspace:*",
@@ -24,24 +24,25 @@
"@openpanel/payments": "workspace:*", "@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*", "@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"bullmq": "^5.63.0", "@openpanel/validation": "workspace:*",
"date-fns": "^3.3.1", "bullmq": "^5.71.1",
"express": "^4.18.2", "date-fns": "^3.6.0",
"express": "^4.22.1",
"groupmq": "catalog:", "groupmq": "catalog:",
"prom-client": "^15.1.3", "prom-client": "^15.1.3",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"source-map-support": "^0.5.21", "source-map-support": "^0.5.21",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"uuid": "^9.0.1" "uuid": "^9.0.1"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/express": "^4.17.21", "@types/express": "^5.0.6",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/source-map-support": "^0.5.10", "@types/source-map-support": "^0.5.10",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"@types/uuid": "^9.0.8", "@types/uuid": "^11.0.0",
"tsdown": "0.14.2", "tsdown": "0.21.7",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -73,6 +73,11 @@ export async function bootCron() {
type: 'flushGroups', type: 'flushGroups',
pattern: 1000 * 10, pattern: 1000 * 10,
}, },
{
name: 'flush',
type: 'flushLogs',
pattern: 1000 * 10,
},
{ {
name: 'insightsDaily', name: 'insightsDaily',
type: 'insightsDaily', type: 'insightsDaily',

View File

@@ -8,6 +8,7 @@ import {
gscQueue, gscQueue,
importQueue, importQueue,
insightsQueue, insightsQueue,
logsQueue,
miscQueue, miscQueue,
notificationQueue, notificationQueue,
queueLogger, queueLogger,
@@ -22,6 +23,7 @@ import { incomingEvent } from './jobs/events.incoming-event';
import { gscJob } from './jobs/gsc'; import { gscJob } from './jobs/gsc';
import { importJob } from './jobs/import'; import { importJob } from './jobs/import';
import { insightsProjectJob } from './jobs/insights'; import { insightsProjectJob } from './jobs/insights';
import { incomingLog } from './jobs/logs.incoming-log';
import { miscJob } from './jobs/misc'; import { miscJob } from './jobs/misc';
import { notificationJob } from './jobs/notification'; import { notificationJob } from './jobs/notification';
import { sessionsJob } from './jobs/sessions'; import { sessionsJob } from './jobs/sessions';
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
'import', 'import',
'insights', 'insights',
'gsc', 'gsc',
'logs',
]; ];
} }
@@ -221,6 +224,20 @@ export function bootWorkers() {
logger.info('Started worker for gsc', { concurrency }); logger.info('Started worker for gsc', { concurrency });
} }
// Start logs worker
if (enabledQueues.includes('logs')) {
const concurrency = getConcurrencyFor('logs', 10);
const logsWorker = new Worker(
logsQueue.name,
async (job) => {
await incomingLog(job.data.payload);
},
{ ...workerOptions, concurrency },
);
workers.push(logsWorker);
logger.info('Started worker for logs', { concurrency });
}
if (workers.length === 0) { if (workers.length === 0) {
logger.warn( logger.warn(
'No workers started. Check ENABLED_QUEUES environment variable.' 'No workers started. Check ENABLED_QUEUES environment variable.'

View File

@@ -1,6 +1,6 @@
import type { Job } from 'bullmq'; import type { Job } from 'bullmq';
import { eventBuffer, groupBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db'; import { eventBuffer, groupBuffer, logBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
import type { CronQueuePayload } from '@openpanel/queue'; import type { CronQueuePayload } from '@openpanel/queue';
import { jobdeleteProjects } from './cron.delete-projects'; import { jobdeleteProjects } from './cron.delete-projects';
@@ -33,6 +33,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
case 'flushGroups': { case 'flushGroups': {
return await groupBuffer.tryFlush(); return await groupBuffer.tryFlush();
} }
case 'flushLogs': {
return await logBuffer.tryFlush();
}
case 'ping': { case 'ping': {
return await ping(); return await ping();
} }

View File

@@ -0,0 +1,63 @@
import type { IClickhouseLog } from '@openpanel/db';
import { logBuffer } from '@openpanel/db';
import type { LogsQueuePayload } from '@openpanel/queue';
import { SEVERITY_TEXT_TO_NUMBER } from '@openpanel/validation';
import { logger as baseLogger } from '@/utils/logger';
export async function incomingLog(
payload: LogsQueuePayload['payload'],
): Promise<void> {
const logger = baseLogger.child({ projectId: payload.projectId });
try {
const { log, uaInfo, geo, deviceId, sessionId, projectId, headers } = payload;
const sdkName = headers['openpanel-sdk-name'] ?? '';
const sdkVersion = headers['openpanel-sdk-version'] ?? '';
const severityNumber =
log.severityNumber ??
SEVERITY_TEXT_TO_NUMBER[log.severity] ??
9; // INFO fallback
const row: IClickhouseLog = {
project_id: projectId,
device_id: deviceId,
profile_id: log.profileId ? String(log.profileId) : '',
session_id: sessionId,
timestamp: log.timestamp,
observed_at: new Date().toISOString(),
severity_number: severityNumber,
severity_text: log.severity,
body: log.body,
trace_id: log.traceId ?? '',
span_id: log.spanId ?? '',
trace_flags: log.traceFlags ?? 0,
logger_name: log.loggerName ?? '',
attributes: log.attributes ?? {},
resource: log.resource ?? {},
sdk_name: sdkName,
sdk_version: sdkVersion,
country: geo.country ?? '',
city: geo.city ?? '',
region: geo.region ?? '',
os: uaInfo.os ?? '',
os_version: uaInfo.osVersion ?? '',
browser: uaInfo.isServer ? '' : (uaInfo.browser ?? ''),
browser_version: uaInfo.isServer ? '' : (uaInfo.browserVersion ?? ''),
device: uaInfo.device ?? '',
brand: uaInfo.isServer ? '' : (uaInfo.brand ?? ''),
model: uaInfo.isServer ? '' : (uaInfo.model ?? ''),
};
logBuffer.add(row);
logger.info('Log queued', {
severity: log.severity,
loggerName: log.loggerName,
});
} catch (error) {
logger.error('Failed to process incoming log', { error });
throw error;
}
}

146
docker-compose.prod.yml Normal file
View File

@@ -0,0 +1,146 @@
services:
op-db:
image: postgres:18.3-alpine
restart: always
volumes:
- op-db-data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
op-kv:
image: redis:8.6.2-alpine
restart: always
volumes:
- op-kv-data:/data
command: ["redis-server", "--maxmemory-policy", "noeviction"]
healthcheck:
test: ["CMD-SHELL", "redis-cli ping"]
interval: 10s
timeout: 5s
retries: 5
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
op-ch:
image: clickhouse/clickhouse-server:26.3.2.3
restart: always
environment:
- CLICKHOUSE_DEFAULT_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse}
volumes:
- op-ch-data:/var/lib/clickhouse
- op-ch-logs:/var/log/clickhouse-server
- ./self-hosting/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/op-config.xml:ro
- ./self-hosting/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro
- ./self-hosting/clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
healthcheck:
test: ["CMD-SHELL", "clickhouse-client --query 'SELECT 1'"]
interval: 10s
timeout: 5s
retries: 5
ulimits:
nofile:
soft: 262144
hard: 262144
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
op-api:
image: git.zias.be/zias/openpanel-api:latest
restart: always
ports:
- "3001:3000"
command: >
sh -c "
echo 'Running migrations...'
CI=true pnpm -r run migrate:deploy
pnpm start
"
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
depends_on:
op-db:
condition: service_healthy
op-ch:
condition: service_healthy
op-kv:
condition: service_healthy
env_file:
- .env.prod
logging:
driver: "json-file"
options:
max-size: "50m"
max-file: "3"
op-dashboard:
image: git.zias.be/zias/openpanel-dashboard:latest
restart: always
ports:
- "3000:3000"
depends_on:
op-api:
condition: service_healthy
env_file:
- .env.prod
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
logging:
driver: "json-file"
options:
max-size: "20m"
max-file: "3"
op-worker:
image: git.zias.be/zias/openpanel-worker:latest
restart: always
ports:
- "3002:3000"
depends_on:
op-api:
condition: service_healthy
env_file:
- .env.prod
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
logging:
driver: "json-file"
options:
max-size: "30m"
max-file: "3"
volumes:
op-db-data:
driver: local
op-kv-data:
driver: local
op-ch-data:
driver: local
op-ch-logs:
driver: local

View File

@@ -2,7 +2,7 @@ version: "3"
services: services:
op-db: op-db:
image: postgres:14-alpine image: postgres:18.3-alpine
restart: always restart: always
volumes: volumes:
- ./docker/data/op-db-data:/var/lib/postgresql/data - ./docker/data/op-db-data:/var/lib/postgresql/data
@@ -13,7 +13,7 @@ services:
- POSTGRES_PASSWORD=postgres - POSTGRES_PASSWORD=postgres
op-kv: op-kv:
image: redis:7.2.5-alpine image: redis:8.6.2-alpine
restart: always restart: always
volumes: volumes:
- ./docker/data/op-kv-data:/data - ./docker/data/op-kv-data:/data
@@ -22,7 +22,7 @@ services:
- 6379:6379 - 6379:6379
op-ch: op-ch:
image: clickhouse/clickhouse-server:26.1.3.52 image: clickhouse/clickhouse-server:26.3.2.3
restart: always restart: always
volumes: volumes:
- ./docker/data/op-ch-data:/var/lib/clickhouse - ./docker/data/op-ch-data:/var/lib/clickhouse

View File

@@ -5,7 +5,7 @@
"private": true, "private": true,
"license": "MIT", "license": "MIT",
"author": "Carl-Gerhard Lindesvärd", "author": "Carl-Gerhard Lindesvärd",
"packageManager": "pnpm@10.6.2", "packageManager": "pnpm@10.33.0",
"scripts": { "scripts": {
"test": "vitest run", "test": "vitest run",
"gen:bots": "pnpm -r --filter api gen:bots", "gen:bots": "pnpm -r --filter api gen:bots",
@@ -30,11 +30,11 @@
"pre-push": "[ -n \"$SKIP_HOOKS\" ] || (pnpm typecheck && pnpm test)" "pre-push": "[ -n \"$SKIP_HOOKS\" ] || (pnpm typecheck && pnpm test)"
}, },
"dependencies": { "dependencies": {
"@hyperdx/node-opentelemetry": "^0.8.1", "@hyperdx/node-opentelemetry": "^0.10.3",
"dotenv-cli": "^7.3.0", "dotenv-cli": "^7.4.4",
"semver": "^7.5.4", "semver": "^7.7.4",
"typescript": "catalog:", "typescript": "catalog:",
"winston": "^3.14.2" "winston": "^3.19.0"
}, },
"trustedDependencies": [ "trustedDependencies": [
"@biomejs/biome", "@biomejs/biome",
@@ -49,15 +49,16 @@
"sharp" "sharp"
], ],
"devDependencies": { "devDependencies": {
"@biomejs/biome": "2.3.15", "@biomejs/biome": "2.4.10",
"depcheck": "^1.4.7", "depcheck": "^1.4.7",
"simple-git-hooks": "^2.12.1", "simple-git-hooks": "^2.13.1",
"ultracite": "7.2.0", "ultracite": "7.4.0",
"vitest": "^3.0.4" "vitest": "^3.2.4"
}, },
"pnpm": { "pnpm": {
"patchedDependencies": { "patchedDependencies": {
"nuqs": "patches/nuqs.patch" "nuqs": "patches/nuqs.patch",
"buffer-equal-constant-time": "patches/buffer-equal-constant-time.patch"
}, },
"overrides": { "overrides": {
"rolldown": "1.0.0-beta.43", "rolldown": "1.0.0-beta.43",

View File

@@ -12,13 +12,13 @@
"@openpanel/validation": "workspace:^", "@openpanel/validation": "workspace:^",
"@oslojs/crypto": "^1.0.1", "@oslojs/crypto": "^1.0.1",
"@oslojs/encoding": "^1.1.0", "@oslojs/encoding": "^1.1.0",
"arctic": "^2.3.0" "arctic": "^2.3.4"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/react": "catalog:", "@types/react": "catalog:",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
}, },
"peerDependencies": { "peerDependencies": {

View File

@@ -15,15 +15,15 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/constants": "workspace:*", "@openpanel/constants": "workspace:*",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"lru-cache": "^11.2.4", "lru-cache": "^11.2.7",
"luxon": "^3.7.2", "luxon": "^3.7.2",
"mathjs": "^12.3.2", "mathjs": "^15.1.1",
"nanoid": "^5.1.6", "nanoid": "^5.1.7",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"slugify": "^1.6.6", "slugify": "^1.6.8",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"ua-parser-js": "^2.0.6", "ua-parser-js": "^2.0.9",
"unique-names-generator": "^4.7.1" "unique-names-generator": "^4.7.1"
}, },
"devDependencies": { "devDependencies": {
@@ -31,9 +31,9 @@
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@types/luxon": "^3.7.1", "@types/luxon": "^3.7.1",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/ua-parser-js": "^0.7.39", "@types/ua-parser-js": "^0.7.39",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -8,7 +8,7 @@
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -0,0 +1,72 @@
import { createTable, runClickhouseMigrationCommands } from '../src/clickhouse/migration';
import { getIsCluster, getIsSelfHosting, printBoxMessage } from './helpers';
export async function up() {
const replicatedVersion = '1';
const isClustered = getIsCluster();
const sqls: string[] = [];
sqls.push(
...createTable({
name: 'logs',
columns: [
'`id` UUID DEFAULT generateUUIDv4()',
'`project_id` String CODEC(ZSTD(3))',
'`device_id` String CODEC(ZSTD(3))',
'`profile_id` String CODEC(ZSTD(3))',
'`session_id` String CODEC(LZ4)',
// OpenTelemetry log fields
'`timestamp` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
'`observed_at` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
'`severity_number` UInt8',
'`severity_text` LowCardinality(String)',
'`body` String CODEC(ZSTD(3))',
'`trace_id` String CODEC(ZSTD(3))',
'`span_id` String CODEC(ZSTD(3))',
'`trace_flags` UInt32 DEFAULT 0',
'`logger_name` LowCardinality(String)',
// OTel attributes (log-level key-value pairs)
'`attributes` Map(String, String) CODEC(ZSTD(3))',
// OTel resource attributes (device/app metadata)
'`resource` Map(String, String) CODEC(ZSTD(3))',
// Server-enriched context
'`sdk_name` LowCardinality(String)',
'`sdk_version` LowCardinality(String)',
'`country` LowCardinality(FixedString(2))',
'`city` String',
'`region` LowCardinality(String)',
'`os` LowCardinality(String)',
'`os_version` LowCardinality(String)',
'`browser` LowCardinality(String)',
'`browser_version` LowCardinality(String)',
'`device` LowCardinality(String)',
'`brand` LowCardinality(String)',
'`model` LowCardinality(String)',
],
indices: [
'INDEX idx_severity_number severity_number TYPE minmax GRANULARITY 1',
'INDEX idx_body body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1',
'INDEX idx_trace_id trace_id TYPE bloom_filter GRANULARITY 1',
'INDEX idx_logger_name logger_name TYPE bloom_filter GRANULARITY 1',
],
orderBy: ['project_id', 'toDate(timestamp)', 'severity_number', 'device_id'],
partitionBy: 'toYYYYMM(timestamp)',
settings: {
index_granularity: 8192,
ttl_only_drop_parts: 1,
},
distributionHash: 'cityHash64(project_id, toString(toStartOfHour(timestamp)))',
replicatedVersion,
isClustered,
}),
);
printBoxMessage('Running migration: 13-add-logs', [
'Creates the logs table for OpenTelemetry-compatible device/app log capture.',
]);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}

View File

@@ -13,7 +13,7 @@
"with-env": "dotenv -e ../../.env -c --" "with-env": "dotenv -e ../../.env -c --"
}, },
"dependencies": { "dependencies": {
"@clickhouse/client": "^1.12.1", "@clickhouse/client": "^1.18.2",
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/constants": "workspace:*", "@openpanel/constants": "workspace:*",
"@openpanel/json": "workspace:*", "@openpanel/json": "workspace:*",
@@ -21,13 +21,13 @@
"@openpanel/queue": "workspace:^", "@openpanel/queue": "workspace:^",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@prisma/client": "^6.14.0", "@prisma/client": "^6.19.2",
"@prisma/extension-read-replicas": "^0.4.1", "@prisma/extension-read-replicas": "^0.5.0",
"fast-deep-equal": "^3.1.3", "fast-deep-equal": "^3.1.3",
"jiti": "^2.4.1", "jiti": "^2.6.1",
"mathjs": "^12.3.2", "mathjs": "^15.1.1",
"prisma-json-types-generator": "^3.1.1", "prisma-json-types-generator": "^3.6.2",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"uuid": "^9.0.1", "uuid": "^9.0.1",
@@ -36,10 +36,10 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"@types/uuid": "^9.0.8", "@types/uuid": "^11.0.0",
"prisma": "^6.14.0", "prisma": "^6.19.2",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -1,6 +1,7 @@
import { BotBuffer as BotBufferRedis } from './bot-buffer'; import { BotBuffer as BotBufferRedis } from './bot-buffer';
import { EventBuffer as EventBufferRedis } from './event-buffer'; import { EventBuffer as EventBufferRedis } from './event-buffer';
import { GroupBuffer } from './group-buffer'; import { GroupBuffer } from './group-buffer';
import { LogBuffer } from './log-buffer';
import { ProfileBackfillBuffer } from './profile-backfill-buffer'; import { ProfileBackfillBuffer } from './profile-backfill-buffer';
import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer'; import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
import { ReplayBuffer } from './replay-buffer'; import { ReplayBuffer } from './replay-buffer';
@@ -13,6 +14,8 @@ export const sessionBuffer = new SessionBuffer();
export const profileBackfillBuffer = new ProfileBackfillBuffer(); export const profileBackfillBuffer = new ProfileBackfillBuffer();
export const replayBuffer = new ReplayBuffer(); export const replayBuffer = new ReplayBuffer();
export const groupBuffer = new GroupBuffer(); export const groupBuffer = new GroupBuffer();
export const logBuffer = new LogBuffer();
export type { ProfileBackfillEntry } from './profile-backfill-buffer'; export type { ProfileBackfillEntry } from './profile-backfill-buffer';
export type { IClickhouseSessionReplayChunk } from './replay-buffer'; export type { IClickhouseSessionReplayChunk } from './replay-buffer';
export type { IClickhouseLog } from './log-buffer';

View File

@@ -0,0 +1,193 @@
import { getSafeJson } from '@openpanel/json';
import { getRedisCache } from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import { BaseBuffer } from './base-buffer';
export interface IClickhouseLog {
id?: string;
project_id: string;
device_id: string;
profile_id: string;
session_id: string;
timestamp: string;
observed_at: string;
severity_number: number;
severity_text: string;
body: string;
trace_id: string;
span_id: string;
trace_flags: number;
logger_name: string;
attributes: Record<string, string>;
resource: Record<string, string>;
sdk_name: string;
sdk_version: string;
country: string;
city: string;
region: string;
os: string;
os_version: string;
browser: string;
browser_version: string;
device: string;
brand: string;
model: string;
}
export class LogBuffer extends BaseBuffer {
private batchSize = process.env.LOG_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.LOG_BUFFER_BATCH_SIZE, 10)
: 4000;
private chunkSize = process.env.LOG_BUFFER_CHUNK_SIZE
? Number.parseInt(process.env.LOG_BUFFER_CHUNK_SIZE, 10)
: 1000;
private microBatchIntervalMs = process.env.LOG_BUFFER_MICRO_BATCH_MS
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_MS, 10)
: 10;
private microBatchMaxSize = process.env.LOG_BUFFER_MICRO_BATCH_SIZE
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_SIZE, 10)
: 100;
private pendingLogs: IClickhouseLog[] = [];
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private isFlushing = false;
private flushRetryCount = 0;
private queueKey = 'log_buffer:queue';
protected bufferCounterKey = 'log_buffer:total_count';
constructor() {
super({
name: 'log',
onFlush: async () => {
await this.processBuffer();
},
});
}
add(log: IClickhouseLog) {
this.pendingLogs.push(log);
if (this.pendingLogs.length >= this.microBatchMaxSize) {
this.flushLocalBuffer();
return;
}
if (!this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
public async flush() {
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
await this.flushLocalBuffer();
}
private async flushLocalBuffer() {
if (this.isFlushing || this.pendingLogs.length === 0) {
return;
}
this.isFlushing = true;
const logsToFlush = this.pendingLogs;
this.pendingLogs = [];
try {
const redis = getRedisCache();
const multi = redis.multi();
for (const log of logsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(log));
}
multi.incrby(this.bufferCounterKey, logsToFlush.length);
await multi.exec();
this.flushRetryCount = 0;
} catch (error) {
this.pendingLogs = logsToFlush.concat(this.pendingLogs);
this.flushRetryCount += 1;
this.logger.warn('Failed to flush log buffer to Redis; logs re-queued', {
error,
logCount: logsToFlush.length,
flushRetryCount: this.flushRetryCount,
});
} finally {
this.isFlushing = false;
if (this.pendingLogs.length > 0 && !this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
}
async processBuffer() {
const redis = getRedisCache();
try {
const queueLogs = await redis.lrange(this.queueKey, 0, this.batchSize - 1);
if (queueLogs.length === 0) {
this.logger.debug('No logs to process');
return;
}
const logsToClickhouse: IClickhouseLog[] = [];
for (const logStr of queueLogs) {
const log = getSafeJson<IClickhouseLog>(logStr);
if (log) {
logsToClickhouse.push(log);
}
}
if (logsToClickhouse.length === 0) {
this.logger.debug('No valid logs to process');
return;
}
logsToClickhouse.sort(
(a, b) =>
new Date(a.timestamp || 0).getTime() -
new Date(b.timestamp || 0).getTime(),
);
this.logger.info('Inserting logs into ClickHouse', {
totalLogs: logsToClickhouse.length,
chunks: Math.ceil(logsToClickhouse.length / this.chunkSize),
});
for (const chunk of this.chunks(logsToClickhouse, this.chunkSize)) {
await ch.insert({
table: 'logs',
values: chunk,
format: 'JSONEachRow',
});
}
await redis
.multi()
.ltrim(this.queueKey, queueLogs.length, -1)
.decrby(this.bufferCounterKey, queueLogs.length)
.exec();
this.logger.info('Processed logs from Redis buffer', {
batchSize: this.batchSize,
logsProcessed: logsToClickhouse.length,
});
} catch (error) {
this.logger.error('Error processing log Redis buffer', { error });
}
}
public getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache();
return await redis.llen(this.queueKey);
});
}
}

View File

@@ -9,10 +9,10 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@react-email/components": "^0.5.6", "@react-email/components": "^1.0.10",
"react": "catalog:", "react": "catalog:",
"react-dom": "catalog:", "react-dom": "catalog:",
"resend": "^4.0.1", "resend": "^4.8.0",
"responsive-react-email": "^0.0.5", "responsive-react-email": "^0.0.5",
"zod": "catalog:" "zod": "catalog:"
}, },

View File

@@ -7,15 +7,15 @@
"codegen": "jiti scripts/download.ts" "codegen": "jiti scripts/download.ts"
}, },
"dependencies": { "dependencies": {
"@maxmind/geoip2-node": "^6.1.0", "@maxmind/geoip2-node": "^6.3.4",
"lru-cache": "^11.2.2" "lru-cache": "^11.2.7"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"fast-extract": "^1.4.3", "fast-extract": "^1.14.2",
"jiti": "^2.4.1", "jiti": "^2.6.1",
"tar": "^7.4.3", "tar": "^7.5.13",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -18,18 +18,18 @@
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/queue": "workspace:*", "@openpanel/queue": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"csv-parse": "^6.1.0", "csv-parse": "^6.2.1",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"uuid": "^9.0.1", "uuid": "^9.0.1",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/logger": "workspace:*", "@openpanel/logger": "workspace:*",
"@types/node": "^20.0.0", "@types/node": "^20.19.37",
"@types/ramda": "^0.31.1", "@types/ramda": "^0.31.1",
"@types/uuid": "^9.0.7", "@types/uuid": "^11.0.0",
"bullmq": "^5.8.7", "bullmq": "^5.71.1",
"typescript": "^5.0.0", "typescript": "^5.9.3",
"vitest": "^1.0.0" "vitest": "^1.6.1"
} }
} }

View File

@@ -7,8 +7,8 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@slack/bolt": "^3.18.0", "@slack/bolt": "^3.22.0",
"@slack/oauth": "^3.0.0" "@slack/oauth": "^3.0.5"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",

View File

@@ -11,11 +11,11 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@babel/parser": "^7.26.0" "@babel/parser": "^7.29.2"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"typescript": "catalog:", "typescript": "catalog:",
"vitest": "^2.1.8" "vitest": "^2.1.9"
} }
} }

View File

@@ -7,13 +7,13 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@hyperdx/node-opentelemetry": "^0.8.1", "@hyperdx/node-opentelemetry": "^0.10.3",
"winston": "^3.14.2" "winston": "^3.19.0"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -11,16 +11,16 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@polar-sh/sdk": "^0.35.4" "@polar-sh/sdk": "^0.46.7"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/inquirer": "^9.0.7", "@types/inquirer": "^9.0.9",
"@types/inquirer-autocomplete-prompt": "^3.0.3", "@types/inquirer-autocomplete-prompt": "^3.0.3",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/react": "catalog:", "@types/react": "catalog:",
"inquirer": "^9.3.5", "inquirer": "^9.3.8",
"inquirer-autocomplete-prompt": "^3.0.1", "inquirer-autocomplete-prompt": "^3.0.1",
"typescript": "catalog:" "typescript": "catalog:"
} }

View File

@@ -10,7 +10,7 @@
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/logger": "workspace:*", "@openpanel/logger": "workspace:*",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"bullmq": "^5.63.0", "bullmq": "^5.71.1",
"groupmq": "catalog:" "groupmq": "catalog:"
}, },
"devDependencies": { "devDependencies": {

View File

@@ -8,7 +8,7 @@ import { createLogger } from '@openpanel/logger';
import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis'; import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
import { Queue } from 'bullmq'; import { Queue } from 'bullmq';
import { Queue as GroupQueue } from 'groupmq'; import { Queue as GroupQueue } from 'groupmq';
import type { ITrackPayload } from '../../validation'; import type { ILogPayload, ITrackPayload } from '../../validation';
export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt( export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(
process.env.EVENTS_GROUP_QUEUES_SHARDS || '1', process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',
@@ -138,6 +138,10 @@ export type CronQueuePayloadFlushGroups = {
type: 'flushGroups'; type: 'flushGroups';
payload: undefined; payload: undefined;
}; };
export type CronQueuePayloadFlushLogs = {
type: 'flushLogs';
payload: undefined;
};
export type CronQueuePayload = export type CronQueuePayload =
| CronQueuePayloadSalt | CronQueuePayloadSalt
| CronQueuePayloadFlushEvents | CronQueuePayloadFlushEvents
@@ -146,6 +150,7 @@ export type CronQueuePayload =
| CronQueuePayloadFlushProfileBackfill | CronQueuePayloadFlushProfileBackfill
| CronQueuePayloadFlushReplay | CronQueuePayloadFlushReplay
| CronQueuePayloadFlushGroups | CronQueuePayloadFlushGroups
| CronQueuePayloadFlushLogs
| CronQueuePayloadPing | CronQueuePayloadPing
| CronQueuePayloadProject | CronQueuePayloadProject
| CronQueuePayloadInsightsDaily | CronQueuePayloadInsightsDaily
@@ -297,3 +302,50 @@ export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {
removeOnFail: 100, removeOnFail: 100,
}, },
}); });
export type LogsQueuePayload = {
type: 'incomingLog';
payload: {
projectId: string;
log: ILogPayload & {
timestamp: string;
};
uaInfo:
| {
readonly isServer: true;
readonly device: 'server';
readonly os: '';
readonly osVersion: '';
readonly browser: '';
readonly browserVersion: '';
readonly brand: '';
readonly model: '';
}
| {
readonly os: string | undefined;
readonly osVersion: string | undefined;
readonly browser: string | undefined;
readonly browserVersion: string | undefined;
readonly device: string;
readonly brand: string | undefined;
readonly model: string | undefined;
readonly isServer: false;
};
geo: {
country: string | undefined;
city: string | undefined;
region: string | undefined;
};
headers: Record<string, string | undefined>;
deviceId: string;
sessionId: string;
};
};
export const logsQueue = new Queue<LogsQueuePayload>(getQueueName('logs'), {
connection: getRedisQueue(),
defaultJobOptions: {
removeOnComplete: 100,
removeOnFail: 1000,
},
});

View File

@@ -8,14 +8,14 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/json": "workspace:*", "@openpanel/json": "workspace:*",
"ioredis": "5.8.2", "ioredis": "5.10.1",
"lru-cache": "^11.2.2" "lru-cache": "^11.2.7"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -9,9 +9,9 @@
"react": "catalog:" "react": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "catalog:",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"prisma": "^5.1.1", "@types/react": "catalog:",
"prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -23,7 +23,7 @@
"@openpanel/web": "workspace:1.3.0-local" "@openpanel/web": "workspace:1.3.0-local"
}, },
"devDependencies": { "devDependencies": {
"astro": "^5.7.7" "astro": "^5.18.1"
}, },
"peerDependencies": { "peerDependencies": {
"astro": "^4.0.0 || ^5.0.0" "astro": "^4.0.0 || ^5.0.0"

View File

@@ -10,17 +10,17 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@openpanel/sdk": "workspace:1.3.0-local", "@openpanel/common": "workspace:*",
"@openpanel/common": "workspace:*" "@openpanel/sdk": "workspace:1.3.0-local"
}, },
"peerDependencies": { "peerDependencies": {
"express": "^4.17.0 || ^5.0.0" "express": "^4.17.0 || ^5.0.0"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/express": "^5.0.3", "@types/express": "^5.0.6",
"@types/request-ip": "^0.0.41", "@types/request-ip": "^0.0.41",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -20,7 +20,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/react": "catalog:", "@types/react": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -31,12 +31,12 @@
"nuxt": "^3.0.0 || ^4.0.0" "nuxt": "^3.0.0 || ^4.0.0"
}, },
"devDependencies": { "devDependencies": {
"@nuxt/kit": "^3.0.0", "@nuxt/kit": "^3.21.2",
"@nuxt/module-builder": "^1.0.2", "@nuxt/module-builder": "^1.0.2",
"@nuxt/types": "^2.18.1", "@nuxt/types": "^2.18.1",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@vue/runtime-core": "^3.5.25", "@vue/runtime-core": "^3.5.31",
"typescript": "catalog:", "typescript": "catalog:",
"unbuild": "^3.6.1" "unbuild": "^3.6.1"
} }

View File

@@ -15,7 +15,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
}, },
"peerDependencies": { "peerDependencies": {

View File

@@ -9,12 +9,11 @@
"build": "rm -rf dist && tsup", "build": "rm -rf dist && tsup",
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": {},
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -7,6 +7,7 @@ import type {
IGroupPayload as GroupPayload, IGroupPayload as GroupPayload,
IIdentifyPayload as IdentifyPayload, IIdentifyPayload as IdentifyPayload,
IIncrementPayload as IncrementPayload, IIncrementPayload as IncrementPayload,
ISeverityText,
ITrackHandlerPayload as TrackHandlerPayload, ITrackHandlerPayload as TrackHandlerPayload,
ITrackPayload as TrackPayload, ITrackPayload as TrackPayload,
} from '@openpanel/validation'; } from '@openpanel/validation';
@@ -19,6 +20,7 @@ export type {
GroupPayload, GroupPayload,
IdentifyPayload, IdentifyPayload,
IncrementPayload, IncrementPayload,
ISeverityText,
TrackHandlerPayload, TrackHandlerPayload,
TrackPayload, TrackPayload,
}; };
@@ -29,6 +31,33 @@ export interface TrackProperties {
groups?: string[]; groups?: string[];
} }
export interface LogProperties {
/** Logger name (e.g. "com.example.MyActivity") */
loggerName?: string;
traceId?: string;
spanId?: string;
traceFlags?: number;
/** Log-level key-value attributes */
attributes?: Record<string, string>;
/** Resource/device attributes */
resource?: Record<string, string>;
/** ISO 8601 timestamp; defaults to now */
timestamp?: string;
}
interface LogPayloadForQueue {
body: string;
severity: ISeverityText;
loggerName?: string;
traceId?: string;
spanId?: string;
traceFlags?: number;
attributes?: Record<string, string>;
resource?: Record<string, string>;
timestamp: string;
profileId?: string;
}
export type UpsertGroupPayload = GroupPayload; export type UpsertGroupPayload = GroupPayload;
export interface OpenPanelOptions { export interface OpenPanelOptions {
@@ -57,6 +86,10 @@ export class OpenPanel {
sessionId?: string; sessionId?: string;
global?: Record<string, unknown>; global?: Record<string, unknown>;
queue: TrackHandlerPayload[] = []; queue: TrackHandlerPayload[] = [];
private logQueue: LogPayloadForQueue[] = [];
private logFlushTimer: ReturnType<typeof setTimeout> | null = null;
private logFlushIntervalMs = 2000;
private logFlushMaxSize = 50;
constructor(options: OpenPanelOptions) { constructor(options: OpenPanelOptions) {
this.options = options; this.options = options;
@@ -327,6 +360,67 @@ export class OpenPanel {
this.queue = remaining; this.queue = remaining;
} }
captureLog(
severity: ISeverityText,
body: string,
properties?: LogProperties,
) {
if (this.options.disabled) {
return;
}
const entry: LogPayloadForQueue = {
body,
severity,
timestamp: properties?.timestamp ?? new Date().toISOString(),
...(this.profileId ? { profileId: this.profileId } : {}),
...(properties?.loggerName ? { loggerName: properties.loggerName } : {}),
...(properties?.traceId ? { traceId: properties.traceId } : {}),
...(properties?.spanId ? { spanId: properties.spanId } : {}),
...(properties?.traceFlags !== undefined
? { traceFlags: properties.traceFlags }
: {}),
...(properties?.attributes ? { attributes: properties.attributes } : {}),
...(properties?.resource ? { resource: properties.resource } : {}),
};
this.logQueue.push(entry);
if (this.logQueue.length >= this.logFlushMaxSize) {
this.flushLogs();
return;
}
if (!this.logFlushTimer) {
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null;
this.flushLogs();
}, this.logFlushIntervalMs);
}
}
private async flushLogs() {
if (this.logFlushTimer) {
clearTimeout(this.logFlushTimer);
this.logFlushTimer = null;
}
if (this.logQueue.length === 0) {
return;
}
const batch = this.logQueue;
this.logQueue = [];
try {
await this.api.fetch('/logs', { logs: batch });
} catch (error) {
this.log('Failed to flush logs', error);
// Re-queue on failure
this.logQueue = batch.concat(this.logQueue);
}
}
log(...args: any[]) { log(...args: any[]) {
if (this.options.debug) { if (this.options.debug) {
console.log('[OpenPanel.dev]', ...args); console.log('[OpenPanel.dev]', ...args);

View File

@@ -17,7 +17,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -1,3 +1,4 @@
export * from './src/root'; export * from './src/root';
export * from './src/trpc'; export * from './src/trpc';
export { getProjectAccess } from './src/access'; export { getProjectAccess } from './src/access';
export type { IServiceLog } from './src/routers/log';

View File

@@ -16,17 +16,17 @@
"@openpanel/integrations": "workspace:^", "@openpanel/integrations": "workspace:^",
"@openpanel/js-runtime": "workspace:*", "@openpanel/js-runtime": "workspace:*",
"@openpanel/payments": "workspace:^", "@openpanel/payments": "workspace:^",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@openpanel/queue": "workspace:*",
"@trpc-limiter/redis": "^0.0.2", "@trpc-limiter/redis": "^0.0.2",
"@trpc/client": "^11.6.0", "@trpc/client": "^11.16.0",
"@trpc/server": "^11.6.0", "@trpc/server": "^11.16.0",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"mathjs": "^12.3.2", "mathjs": "^15.1.1",
"prisma-error-enum": "^0.1.3", "prisma-error-enum": "^0.1.3",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"short-unique-id": "^5.0.3", "short-unique-id": "^5.3.2",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"uuid": "^9.0.1", "uuid": "^9.0.1",
@@ -35,9 +35,9 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -8,6 +8,7 @@ import { eventRouter } from './routers/event';
import { groupRouter } from './routers/group'; import { groupRouter } from './routers/group';
import { gscRouter } from './routers/gsc'; import { gscRouter } from './routers/gsc';
import { importRouter } from './routers/import'; import { importRouter } from './routers/import';
import { logRouter } from './routers/log';
import { insightRouter } from './routers/insight'; import { insightRouter } from './routers/insight';
import { integrationRouter } from './routers/integration'; import { integrationRouter } from './routers/integration';
import { notificationRouter } from './routers/notification'; import { notificationRouter } from './routers/notification';
@@ -57,6 +58,7 @@ export const appRouter = createTRPCRouter({
email: emailRouter, email: emailRouter,
gsc: gscRouter, gsc: gscRouter,
group: groupRouter, group: groupRouter,
log: logRouter,
}); });
// export type definition of API // export type definition of API

View File

@@ -0,0 +1,212 @@
import { chQuery, convertClickhouseDateToJs } from '@openpanel/db';
import { zSeverityText } from '@openpanel/validation';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../trpc';
export interface IServiceLog {
id: string;
projectId: string;
deviceId: string;
profileId: string;
sessionId: string;
timestamp: Date;
severityNumber: number;
severityText: string;
body: string;
traceId: string;
spanId: string;
traceFlags: number;
loggerName: string;
attributes: Record<string, string>;
resource: Record<string, string>;
sdkName: string;
sdkVersion: string;
country: string;
city: string;
region: string;
os: string;
osVersion: string;
browser: string;
browserVersion: string;
device: string;
brand: string;
model: string;
}
interface IClickhouseLog {
id: string;
project_id: string;
device_id: string;
profile_id: string;
session_id: string;
timestamp: string;
severity_number: number;
severity_text: string;
body: string;
trace_id: string;
span_id: string;
trace_flags: number;
logger_name: string;
attributes: Record<string, string>;
resource: Record<string, string>;
sdk_name: string;
sdk_version: string;
country: string;
city: string;
region: string;
os: string;
os_version: string;
browser: string;
browser_version: string;
device: string;
brand: string;
model: string;
}
function toServiceLog(row: IClickhouseLog): IServiceLog {
return {
id: row.id,
projectId: row.project_id,
deviceId: row.device_id,
profileId: row.profile_id,
sessionId: row.session_id,
timestamp: convertClickhouseDateToJs(row.timestamp),
severityNumber: row.severity_number,
severityText: row.severity_text,
body: row.body,
traceId: row.trace_id,
spanId: row.span_id,
traceFlags: row.trace_flags,
loggerName: row.logger_name,
attributes: row.attributes,
resource: row.resource,
sdkName: row.sdk_name,
sdkVersion: row.sdk_version,
country: row.country,
city: row.city,
region: row.region,
os: row.os,
osVersion: row.os_version,
browser: row.browser,
browserVersion: row.browser_version,
device: row.device,
brand: row.brand,
model: row.model,
};
}
export const logRouter = createTRPCRouter({
list: protectedProcedure
.input(
z.object({
projectId: z.string(),
cursor: z.string().nullish(),
severity: z.array(zSeverityText).optional(),
search: z.string().optional(),
loggerName: z.string().optional(),
startDate: z.date().optional(),
endDate: z.date().optional(),
take: z.number().default(50),
}),
)
.query(async ({ input }) => {
const { projectId, cursor, severity, search, loggerName, startDate, endDate, take } = input;
const conditions: string[] = [
`project_id = ${sqlstring.escape(projectId)}`,
];
if (cursor) {
conditions.push(`timestamp < ${sqlstring.escape(cursor)}`);
}
if (severity && severity.length > 0) {
const escaped = severity.map((s) => sqlstring.escape(s)).join(', ');
conditions.push(`severity_text IN (${escaped})`);
}
if (search) {
conditions.push(`body ILIKE ${sqlstring.escape(`%${search}%`)}`);
}
if (loggerName) {
conditions.push(`logger_name = ${sqlstring.escape(loggerName)}`);
}
if (startDate) {
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
}
if (endDate) {
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
}
const where = conditions.join(' AND ');
const rows = await chQuery<IClickhouseLog>(
`SELECT
id, project_id, device_id, profile_id, session_id,
timestamp, severity_number, severity_text, body,
trace_id, span_id, trace_flags, logger_name,
attributes, resource,
sdk_name, sdk_version,
country, city, region, os, os_version,
browser, browser_version, device, brand, model
FROM logs
WHERE ${where}
ORDER BY timestamp DESC
LIMIT ${take + 1}`,
);
const hasMore = rows.length > take;
const data = rows.slice(0, take).map(toServiceLog);
const lastItem = data[data.length - 1];
return {
data,
meta: {
next: hasMore && lastItem ? lastItem.timestamp.toISOString() : null,
},
};
}),
severityCounts: protectedProcedure
.input(
z.object({
projectId: z.string(),
startDate: z.date().optional(),
endDate: z.date().optional(),
}),
)
.query(async ({ input }) => {
const { projectId, startDate, endDate } = input;
const conditions: string[] = [
`project_id = ${sqlstring.escape(projectId)}`,
];
if (startDate) {
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
}
if (endDate) {
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
}
const where = conditions.join(' AND ');
const rows = await chQuery<{ severity_text: string; count: number }>(
`SELECT severity_text, count() AS count
FROM logs
WHERE ${where}
GROUP BY severity_text
ORDER BY count DESC`,
);
return rows.reduce<Record<string, number>>((acc, row) => {
acc[row.severity_text] = row.count;
return acc;
}, {});
}),
});

View File

@@ -13,7 +13,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -626,3 +626,4 @@ export type ICreateImport = z.infer<typeof zCreateImport>;
export * from './types.insights'; export * from './types.insights';
export * from './track.validation'; export * from './track.validation';
export * from './event-blocklist'; export * from './event-blocklist';
export * from './log.validation';

View File

@@ -0,0 +1,60 @@
import { z } from 'zod';
/**
* OTel severity number mapping (subset):
* TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21
*/
export const SEVERITY_TEXT_TO_NUMBER: Record<string, number> = {
trace: 1,
debug: 5,
info: 9,
warn: 13,
warning: 13,
error: 17,
fatal: 21,
critical: 21,
};
export const zSeverityText = z.enum([
'trace',
'debug',
'info',
'warn',
'warning',
'error',
'fatal',
'critical',
]);
export type ISeverityText = z.infer<typeof zSeverityText>;
export const zLogPayload = z.object({
/** Log message / body */
body: z.string().min(1),
/** Severity level as text */
severity: zSeverityText.default('info'),
/** Optional override for the numeric OTel severity (1-24) */
severityNumber: z.number().int().min(1).max(24).optional(),
/** ISO 8601 timestamp; defaults to server receive time if omitted */
timestamp: z.string().datetime({ offset: true }).optional(),
/** Logger name (e.g. "com.example.MyActivity") */
loggerName: z.string().optional(),
/** W3C trace context */
traceId: z.string().optional(),
spanId: z.string().optional(),
traceFlags: z.number().int().min(0).optional(),
/** Log-level key-value attributes */
attributes: z.record(z.string(), z.string()).optional(),
/** Resource/device attributes (app version, runtime, etc.) */
resource: z.record(z.string(), z.string()).optional(),
/** Profile/user ID to associate with this log */
profileId: z.union([z.string().min(1), z.number()]).optional(),
});
export type ILogPayload = z.infer<typeof zLogPayload>;
export const zLogBatchPayload = z.object({
logs: z.array(zLogPayload).min(1).max(500),
});
export type ILogBatchPayload = z.infer<typeof zLogBatchPayload>;

View File

@@ -0,0 +1,42 @@
diff --git a/.npmignore b/.npmignore
deleted file mode 100644
index 34e4f5c298de294fa5c1c1769b6489eb047bde9a..0000000000000000000000000000000000000000
diff --git a/index.js b/index.js
index 5462c1f830bdbe79bf2b1fcfd811cd9799b4dd11..689421d49e83d168981a0c7d1fef59a0b0e56963 100644
--- a/index.js
+++ b/index.js
@@ -3,6 +3,9 @@
var Buffer = require('buffer').Buffer; // browserify
var SlowBuffer = require('buffer').SlowBuffer;
+// Handle Node.js v25+ where SlowBuffer was removed
+var hasSlowBuffer = !!SlowBuffer;
+
module.exports = bufferEq;
function bufferEq(a, b) {
@@ -28,14 +31,18 @@ function bufferEq(a, b) {
}
bufferEq.install = function() {
- Buffer.prototype.equal = SlowBuffer.prototype.equal = function equal(that) {
- return bufferEq(this, that);
- };
+ Buffer.prototype.equal = function equal(that) {
+ return bufferEq(this, that);
+ };
+ if (hasSlowBuffer) {
+ SlowBuffer.prototype.equal = Buffer.prototype.equal;
+ }
};
var origBufEqual = Buffer.prototype.equal;
-var origSlowBufEqual = SlowBuffer.prototype.equal;
bufferEq.restore = function() {
- Buffer.prototype.equal = origBufEqual;
- SlowBuffer.prototype.equal = origSlowBufEqual;
+ Buffer.prototype.equal = origBufEqual;
+ if (hasSlowBuffer && SlowBuffer.prototype) {
+ delete SlowBuffer.prototype.equal;
+ }
};

28713
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,10 +7,10 @@ packages:
# Define a catalog of version ranges. # Define a catalog of version ranges.
catalog: catalog:
zod: ^3.24.2 zod: ^3.24.2
react: ^19.2.3 react: ^19.2.4
"@types/react": ^19.2.3 "@types/react": ^19.2.14
"react-dom": ^19.2.3 "react-dom": ^19.2.4
"@types/react-dom": ^19.2.3 "@types/react-dom": ^19.2.3
"@types/node": ^24.7.1 "@types/node": ^25.5.0
typescript: ^5.9.3 typescript: ^5.9.3
groupmq: 2.0.0-next.1 groupmq: 2.0.0-next.1

View File

@@ -23,7 +23,7 @@ services:
max-file: "3" max-file: "3"
op-db: op-db:
image: postgres:14-alpine image: postgres:18.3-alpine
restart: always restart: always
volumes: volumes:
- op-db-data:/var/lib/postgresql/data - op-db-data:/var/lib/postgresql/data
@@ -45,7 +45,7 @@ services:
# - 5432:5432 # - 5432:5432
op-kv: op-kv:
image: redis:7.2.5-alpine image: redis:8.6.2-alpine
restart: always restart: always
volumes: volumes:
- op-kv-data:/data - op-kv-data:/data
@@ -65,7 +65,7 @@ services:
# - 6379:6379 # - 6379:6379
op-ch: op-ch:
image: clickhouse/clickhouse-server:25.10.2.65 image: clickhouse/clickhouse-server:26.3.2.3
restart: always restart: always
volumes: volumes:
- op-ch-data:/var/lib/clickhouse - op-ch-data:/var/lib/clickhouse

View File

@@ -9,12 +9,12 @@
}, },
"dependencies": { "dependencies": {
"arg": "^5.0.2", "arg": "^5.0.2",
"semver": "^7.5.4" "semver": "^7.7.4"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/semver": "^7.5.4", "@types/semver": "^7.7.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -3,8 +3,12 @@
"version": "0.1.0", "version": "0.1.0",
"type": "module", "type": "module",
"private": true, "private": true,
"files": ["base.json", "sdk.json", "tsup.config.json"], "files": [
"base.json",
"sdk.json",
"tsup.config.json"
],
"devDependencies": { "devDependencies": {
"tsup": "^7.2.0" "tsup": "^8.5.1"
} }
} }