4 Commits

Author SHA1 Message Date
be64494aa9 feat: add Gitea CI workflows, production compose, and update all deps
Some checks failed
Build and Push API / build-api (push) Failing after 39m34s
Build and Push Dashboard / build-dashboard (push) Has been cancelled
Build and Push Worker / build-worker (push) Has been cancelled
- Add .gitea/workflows for building and pushing api, dashboard, and worker images to git.zias.be registry
- Add docker-compose.prod.yml using pre-built registry images (no build-from-source on server)
- Update docker-compose.yml infra images to latest (postgres 18.3, redis 8.6.2, clickhouse 26.3.2.3)
- Update self-hosting/docker-compose.template.yml image versions to match
- Bump Node.js to 22.22.2 in all three Dockerfiles
- Update pnpm to 10.33.0 and upgrade all safe npm dependencies
- Add buffer-equal-constant-time patch

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-31 09:18:55 +02:00
eefbeac7f8 feat: add logs UI — tRPC router, page, and sidebar nav
- logRouter with list (infinite query, cursor-based) and severityCounts procedures
- /logs route page: severity filter chips with counts, search input, expandable log rows
- Log detail expansion shows attributes, resource, trace context, device/location
- Sidebar "Logs" link with ScrollText icon between Events and Sessions

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-30 12:08:39 +02:00
0672857974 feat: add OpenTelemetry device log capture pipeline
- ClickHouse `logs` table (migration 13) with OTel columns, bloom filter indices
- Zod validation schema for log payloads (severity, body, attributes, trace context)
- Redis-backed LogBuffer with micro-batching into ClickHouse
- POST /logs API endpoint with client auth, geo + UA enrichment
- BullMQ logs queue + worker job
- cron flushLogs every 10s wired into existing cron system
- SDK captureLog(severity, body, properties) with client-side batching

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-30 12:04:04 +02:00
Carl-Gerhard Lindesvärd
a1ce71ffb6 fix:buffers
* wip

* remove active visitor counter in redis

* test

* fix profiel query

* fix
2026-03-24 13:54:00 +01:00
76 changed files with 15276 additions and 16739 deletions

View File

@@ -0,0 +1,55 @@
name: Build and Push API
on:
push:
branches: [main]
tags: ["v*"]
pull_request:
branches: [main]
env:
REGISTRY: git.zias.be
OWNER: zias
jobs:
build-api:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ env.OWNER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=sha,prefix=sha-,format=short
type=semver,pattern={{version}}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
file: apps/api/Dockerfile
target: runner
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api:buildcache
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-api:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
build-args: |
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres

View File

@@ -0,0 +1,53 @@
name: Build and Push Dashboard
on:
push:
branches: [main]
tags: ["v*"]
pull_request:
branches: [main]
env:
REGISTRY: git.zias.be
OWNER: zias
jobs:
build-dashboard:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ env.OWNER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=sha,prefix=sha-,format=short
type=semver,pattern={{version}}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
file: apps/start/Dockerfile
target: runner
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard:buildcache
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-dashboard:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}

View File

@@ -0,0 +1,55 @@
name: Build and Push Worker
on:
push:
branches: [main]
tags: ["v*"]
pull_request:
branches: [main]
env:
REGISTRY: git.zias.be
OWNER: zias
jobs:
build-worker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ env.OWNER }}
password: ${{ secrets.REGISTRY_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=sha,prefix=sha-,format=short
type=semver,pattern={{version}}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
file: apps/worker/Dockerfile
target: runner
platforms: linux/amd64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker:buildcache
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-worker:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
build-args: |
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres

View File

@@ -10,14 +10,14 @@
"dependencies": { "dependencies": {
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"chalk": "^5.3.0", "chalk": "^5.6.2",
"fuzzy": "^0.1.3", "fuzzy": "^0.1.3",
"inquirer": "^9.3.5", "inquirer": "^9.3.8",
"inquirer-autocomplete-prompt": "^3.0.1", "inquirer-autocomplete-prompt": "^3.0.1",
"jiti": "^2.4.2" "jiti": "^2.6.1"
}, },
"devDependencies": { "devDependencies": {
"@types/inquirer": "^9.0.7", "@types/inquirer": "^9.0.9",
"@types/inquirer-autocomplete-prompt": "^3.0.3", "@types/inquirer-autocomplete-prompt": "^3.0.3",
"@types/node": "catalog:", "@types/node": "catalog:",
"typescript": "catalog:" "typescript": "catalog:"

View File

@@ -1,4 +1,4 @@
ARG NODE_VERSION=22.20.0 ARG NODE_VERSION=22.22.2
FROM node:${NODE_VERSION}-slim AS base FROM node:${NODE_VERSION}-slim AS base

View File

@@ -12,11 +12,11 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@ai-sdk/anthropic": "^1.2.10", "@ai-sdk/anthropic": "^1.2.12",
"@ai-sdk/openai": "^1.3.12", "@ai-sdk/openai": "^1.3.24",
"@fastify/compress": "^8.1.0", "@fastify/compress": "^8.3.1",
"@fastify/cookie": "^11.0.2", "@fastify/cookie": "^11.0.2",
"@fastify/cors": "^11.1.0", "@fastify/cors": "^11.2.0",
"@fastify/rate-limit": "^10.3.0", "@fastify/rate-limit": "^10.3.0",
"@fastify/websocket": "^11.2.0", "@fastify/websocket": "^11.2.0",
"@node-rs/argon2": "^2.0.2", "@node-rs/argon2": "^2.0.2",
@@ -33,36 +33,36 @@
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"@openpanel/trpc": "workspace:*", "@openpanel/trpc": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@trpc/server": "^11.6.0", "@trpc/server": "^11.16.0",
"ai": "^4.2.10", "ai": "^4.3.19",
"fast-json-stable-hash": "^1.0.3", "fast-json-stable-hash": "^1.0.3",
"fastify": "^5.6.1", "fastify": "^5.8.4",
"fastify-metrics": "^12.1.0", "fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0", "fastify-raw-body": "^5.0.0",
"groupmq": "catalog:", "groupmq": "catalog:",
"jsonwebtoken": "^9.0.2", "jsonwebtoken": "^9.0.3",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"sharp": "^0.33.5", "sharp": "^0.34.5",
"source-map-support": "^0.5.21", "source-map-support": "^0.5.21",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"svix": "^1.24.0", "svix": "^1.89.0",
"url-metadata": "^5.4.1", "url-metadata": "^5.4.3",
"uuid": "^9.0.1", "uuid": "^9.0.1",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@faker-js/faker": "^9.0.1", "@faker-js/faker": "^9.9.0",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/js-yaml": "^4.0.9", "@types/js-yaml": "^4.0.9",
"@types/jsonwebtoken": "^9.0.9", "@types/jsonwebtoken": "^9.0.10",
"@types/ramda": "^0.30.2", "@types/ramda": "^0.31.1",
"@types/source-map-support": "^0.5.10", "@types/source-map-support": "^0.5.10",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"@types/uuid": "^10.0.0", "@types/uuid": "^11.0.0",
"@types/ws": "^8.5.14", "@types/ws": "^8.18.1",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.1",
"tsdown": "0.14.2", "tsdown": "0.21.7",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -1,10 +1,7 @@
import type { WebSocket } from '@fastify/websocket'; import type { WebSocket } from '@fastify/websocket';
import { eventBuffer } from '@openpanel/db'; import { eventBuffer } from '@openpanel/db';
import { setSuperJson } from '@openpanel/json'; import { setSuperJson } from '@openpanel/json';
import { import { subscribeToPublishedEvent } from '@openpanel/redis';
psubscribeToPublishedEvent,
subscribeToPublishedEvent,
} from '@openpanel/redis';
import { getProjectAccess } from '@openpanel/trpc'; import { getProjectAccess } from '@openpanel/trpc';
import { getOrganizationAccess } from '@openpanel/trpc/src/access'; import { getOrganizationAccess } from '@openpanel/trpc/src/access';
import type { FastifyRequest } from 'fastify'; import type { FastifyRequest } from 'fastify';
@@ -39,19 +36,8 @@ export function wsVisitors(
} }
); );
const punsubscribe = psubscribeToPublishedEvent(
'__keyevent@0__:expired',
(key) => {
const [, , projectId] = key.split(':');
if (projectId === params.projectId) {
sendCount();
}
}
);
socket.on('close', () => { socket.on('close', () => {
unsubscribe(); unsubscribe();
punsubscribe();
}); });
} }

View File

@@ -0,0 +1,68 @@
import { parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { getGeoLocation } from '@openpanel/geo';
import { type LogsQueuePayload, logsQueue } from '@openpanel/queue';
import { type ILogBatchPayload, zLogBatchPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { getDeviceId } from '@/utils/ids';
import { getStringHeaders } from './track.controller';
export async function handler(
request: FastifyRequest<{ Body: ILogBatchPayload }>,
reply: FastifyReply,
) {
const projectId = request.client?.projectId;
if (!projectId) {
return reply.status(400).send({ status: 400, error: 'Missing projectId' });
}
const validationResult = zLogBatchPayload.safeParse(request.body);
if (!validationResult.success) {
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Validation failed',
errors: validationResult.error.errors,
});
}
const { logs } = validationResult.data;
const ip = request.clientIp;
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
const headers = getStringHeaders(request.headers);
const receivedAt = new Date().toISOString();
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
const { deviceId, sessionId } = await getDeviceId({ projectId, ip, ua, salts });
const uaInfo = parseUserAgent(ua, undefined);
const jobs: LogsQueuePayload[] = logs.map((log) => ({
type: 'incomingLog' as const,
payload: {
projectId,
log: {
...log,
timestamp: log.timestamp ?? receivedAt,
},
uaInfo,
geo: {
country: geo.country,
city: geo.city,
region: geo.region,
},
headers,
deviceId,
sessionId,
},
}));
await logsQueue.addBulk(
jobs.map((job) => ({
name: 'incomingLog',
data: job,
})),
);
return reply.status(200).send({ ok: true, count: logs.length });
}

View File

@@ -44,6 +44,7 @@ import manageRouter from './routes/manage.router';
import miscRouter from './routes/misc.router'; import miscRouter from './routes/misc.router';
import oauthRouter from './routes/oauth-callback.router'; import oauthRouter from './routes/oauth-callback.router';
import profileRouter from './routes/profile.router'; import profileRouter from './routes/profile.router';
import logsRouter from './routes/logs.router';
import trackRouter from './routes/track.router'; import trackRouter from './routes/track.router';
import webhookRouter from './routes/webhook.router'; import webhookRouter from './routes/webhook.router';
import { HttpError } from './utils/errors'; import { HttpError } from './utils/errors';
@@ -209,6 +210,7 @@ const startServer = async () => {
instance.register(importRouter, { prefix: '/import' }); instance.register(importRouter, { prefix: '/import' });
instance.register(insightsRouter, { prefix: '/insights' }); instance.register(insightsRouter, { prefix: '/insights' });
instance.register(trackRouter, { prefix: '/track' }); instance.register(trackRouter, { prefix: '/track' });
instance.register(logsRouter, { prefix: '/logs' });
instance.register(manageRouter, { prefix: '/manage' }); instance.register(manageRouter, { prefix: '/manage' });
// Keep existing endpoints for backward compatibility // Keep existing endpoints for backward compatibility
instance.get('/healthcheck', healthcheck); instance.get('/healthcheck', healthcheck);

View File

@@ -0,0 +1,17 @@
import type { FastifyPluginCallback } from 'fastify';
import { handler } from '@/controllers/logs.controller';
import { clientHook } from '@/hooks/client.hook';
import { duplicateHook } from '@/hooks/duplicate.hook';
const logsRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preValidation', duplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.route({
method: 'POST',
url: '/',
handler,
});
};
export default logsRouter;

View File

@@ -16,11 +16,11 @@
}, },
"dependencies": { "dependencies": {
"@nivo/funnel": "^0.99.0", "@nivo/funnel": "^0.99.0",
"@number-flow/react": "0.5.10", "@number-flow/react": "0.6.0",
"@opennextjs/cloudflare": "^1.17.1", "@opennextjs/cloudflare": "^1.18.0",
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/geo": "workspace:*", "@openpanel/geo": "workspace:*",
"@openpanel/nextjs": "^1.2.0", "@openpanel/nextjs": "^1.4.0",
"@openpanel/payments": "workspace:^", "@openpanel/payments": "workspace:^",
"@openpanel/sdk-info": "workspace:^", "@openpanel/sdk-info": "workspace:^",
"@openstatus/react": "0.0.3", "@openstatus/react": "0.0.3",
@@ -28,37 +28,37 @@
"@radix-ui/react-slider": "1.3.6", "@radix-ui/react-slider": "1.3.6",
"@radix-ui/react-slot": "1.2.4", "@radix-ui/react-slot": "1.2.4",
"@radix-ui/react-tooltip": "1.2.8", "@radix-ui/react-tooltip": "1.2.8",
"cheerio": "^1.0.0", "cheerio": "^1.2.0",
"class-variance-authority": "0.7.1", "class-variance-authority": "0.7.1",
"clsx": "2.1.1", "clsx": "2.1.1",
"dotted-map": "2.2.3", "dotted-map": "3.1.0",
"framer-motion": "12.23.25", "framer-motion": "12.38.0",
"fumadocs-core": "16.2.2", "fumadocs-core": "16.7.7",
"fumadocs-mdx": "14.0.4", "fumadocs-mdx": "14.2.11",
"fumadocs-ui": "16.2.2", "fumadocs-ui": "16.7.7",
"geist": "1.5.1", "geist": "1.7.0",
"lucide-react": "^0.555.0", "lucide-react": "^0.555.0",
"next": "16.0.7", "next": "16.2.1",
"next-themes": "^0.4.6", "next-themes": "^0.4.6",
"react": "catalog:", "react": "catalog:",
"react-dom": "catalog:", "react-dom": "catalog:",
"react-markdown": "^10.1.0", "react-markdown": "^10.1.0",
"recharts": "^2.15.0", "recharts": "^2.15.4",
"rehype-external-links": "3.0.0", "rehype-external-links": "3.0.0",
"tailwind-merge": "3.4.0", "tailwind-merge": "3.5.0",
"tailwindcss-animate": "1.0.7", "tailwindcss-animate": "1.0.7",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@tailwindcss/postcss": "^4.1.17", "@tailwindcss/postcss": "^4.2.2",
"@types/mdx": "^2.0.13", "@types/mdx": "^2.0.13",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/react": "catalog:", "@types/react": "catalog:",
"@types/react-dom": "catalog:", "@types/react-dom": "catalog:",
"autoprefixer": "^10.4.22", "autoprefixer": "^10.4.27",
"postcss": "^8.5.6", "postcss": "^8.5.8",
"tailwindcss": "4.1.17", "tailwindcss": "4.2.2",
"typescript": "catalog:", "typescript": "catalog:",
"wrangler": "^4.65.0" "wrangler": "^4.78.0"
} }
} }

View File

@@ -1,4 +1,4 @@
ARG NODE_VERSION=22.20.0 ARG NODE_VERSION=22.22.2
FROM node:${NODE_VERSION}-slim AS base FROM node:${NODE_VERSION}-slim AS base

View File

@@ -17,21 +17,21 @@
"with-env": "dotenv -e ../../.env -c --" "with-env": "dotenv -e ../../.env -c --"
}, },
"dependencies": { "dependencies": {
"@ai-sdk/react": "^1.2.5", "@ai-sdk/react": "^1.2.12",
"@codemirror/commands": "^6.7.0", "@codemirror/commands": "^6.10.3",
"@codemirror/lang-javascript": "^6.2.0", "@codemirror/lang-javascript": "^6.2.5",
"@codemirror/lang-json": "^6.0.1", "@codemirror/lang-json": "^6.0.2",
"@codemirror/state": "^6.4.0", "@codemirror/state": "^6.6.0",
"@codemirror/theme-one-dark": "^6.1.3", "@codemirror/theme-one-dark": "^6.1.3",
"@codemirror/view": "^6.35.0", "@codemirror/view": "^6.40.0",
"@dnd-kit/core": "^6.3.1", "@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0", "@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2", "@dnd-kit/utilities": "^3.2.2",
"@faker-js/faker": "^9.6.0", "@faker-js/faker": "^9.9.0",
"@hookform/resolvers": "^3.3.4", "@hookform/resolvers": "^3.10.0",
"@hyperdx/node-opentelemetry": "^0.8.1", "@hyperdx/node-opentelemetry": "^0.10.3",
"@nivo/sankey": "^0.99.0", "@nivo/sankey": "^0.99.0",
"@number-flow/react": "0.5.10", "@number-flow/react": "0.6.0",
"@openpanel/common": "workspace:^", "@openpanel/common": "workspace:^",
"@openpanel/constants": "workspace:^", "@openpanel/constants": "workspace:^",
"@openpanel/importer": "workspace:^", "@openpanel/importer": "workspace:^",
@@ -40,100 +40,100 @@
"@openpanel/payments": "workspace:*", "@openpanel/payments": "workspace:*",
"@openpanel/sdk-info": "workspace:^", "@openpanel/sdk-info": "workspace:^",
"@openpanel/validation": "workspace:^", "@openpanel/validation": "workspace:^",
"@openpanel/web": "^1.0.1", "@openpanel/web": "^1.3.0",
"@radix-ui/react-accordion": "^1.2.12", "@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-alert-dialog": "^1.1.15",
"@radix-ui/react-aspect-ratio": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.8",
"@radix-ui/react-avatar": "^1.1.10", "@radix-ui/react-avatar": "^1.1.11",
"@radix-ui/react-checkbox": "^1.3.3", "@radix-ui/react-checkbox": "^1.3.3",
"@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dialog": "^1.1.15",
"@radix-ui/react-dropdown-menu": "^2.1.16", "@radix-ui/react-dropdown-menu": "^2.1.16",
"@radix-ui/react-label": "^2.1.7", "@radix-ui/react-label": "^2.1.8",
"@radix-ui/react-popover": "^1.1.15", "@radix-ui/react-popover": "^1.1.15",
"@radix-ui/react-portal": "^1.1.9", "@radix-ui/react-portal": "^1.1.10",
"@radix-ui/react-progress": "^1.1.7", "@radix-ui/react-progress": "^1.1.8",
"@radix-ui/react-radio-group": "^1.3.8", "@radix-ui/react-radio-group": "^1.3.8",
"@radix-ui/react-scroll-area": "^1.2.10", "@radix-ui/react-scroll-area": "^1.2.10",
"@radix-ui/react-select": "^2.2.6", "@radix-ui/react-select": "^2.2.6",
"@radix-ui/react-separator": "^1.1.7", "@radix-ui/react-separator": "^1.1.8",
"@radix-ui/react-slider": "1.2.3", "@radix-ui/react-slider": "1.3.6",
"@radix-ui/react-slot": "^1.2.3", "@radix-ui/react-slot": "^1.2.4",
"@radix-ui/react-switch": "^1.2.6", "@radix-ui/react-switch": "^1.2.6",
"@radix-ui/react-tabs": "^1.1.13", "@radix-ui/react-tabs": "^1.1.13",
"@radix-ui/react-toast": "^1.2.15", "@radix-ui/react-toast": "^1.2.15",
"@radix-ui/react-toggle": "^1.1.10", "@radix-ui/react-toggle": "^1.1.10",
"@radix-ui/react-toggle-group": "^1.1.11", "@radix-ui/react-toggle-group": "^1.1.11",
"@radix-ui/react-tooltip": "^1.2.8", "@radix-ui/react-tooltip": "^1.2.8",
"@reduxjs/toolkit": "^2.8.2", "@reduxjs/toolkit": "^2.11.2",
"@sentry/tanstackstart-react": "^9.12.0", "@sentry/tanstackstart-react": "^10.46.0",
"@tailwindcss/container-queries": "^0.1.1", "@tailwindcss/container-queries": "^0.1.1",
"@tailwindcss/typography": "^0.5.15", "@tailwindcss/typography": "^0.5.19",
"@tailwindcss/vite": "^4.0.6", "@tailwindcss/vite": "^4.2.2",
"@tanstack/match-sorter-utils": "^8.19.4", "@tanstack/match-sorter-utils": "^8.19.4",
"@tanstack/nitro-v2-vite-plugin": "^1.133.19", "@tanstack/nitro-v2-vite-plugin": "^1.154.9",
"@tanstack/react-devtools": "^0.7.6", "@tanstack/react-devtools": "^0.10.0",
"@tanstack/react-query": "^5.90.2", "@tanstack/react-query": "^5.95.2",
"@tanstack/react-query-devtools": "^5.90.2", "@tanstack/react-query-devtools": "^5.95.2",
"@tanstack/react-router": "^1.132.47", "@tanstack/react-router": "^1.168.10",
"@tanstack/react-router-devtools": "^1.132.51", "@tanstack/react-router-devtools": "^1.166.11",
"@tanstack/react-router-ssr-query": "^1.132.47", "@tanstack/react-router-ssr-query": "^1.166.10",
"@tanstack/react-start": "^1.132.56", "@tanstack/react-start": "^1.167.16",
"@tanstack/react-store": "^0.8.0", "@tanstack/react-store": "^0.9.3",
"@tanstack/react-table": "^8.21.3", "@tanstack/react-table": "^8.21.3",
"@tanstack/react-virtual": "^3.13.12", "@tanstack/react-virtual": "^3.13.23",
"@tanstack/router-plugin": "^1.132.56", "@tanstack/router-plugin": "^1.167.12",
"@tanstack/store": "^0.8.0", "@tanstack/store": "^0.9.3",
"@trpc/client": "^11.6.0", "@trpc/client": "^11.16.0",
"@trpc/react-query": "^11.6.0", "@trpc/react-query": "^11.16.0",
"@trpc/server": "^11.6.0", "@trpc/server": "^11.16.0",
"@trpc/tanstack-react-query": "^11.6.0", "@trpc/tanstack-react-query": "^11.16.0",
"@types/d3": "^7.4.3", "@types/d3": "^7.4.3",
"ai": "^4.2.10", "ai": "^4.3.19",
"bind-event-listener": "^3.0.0", "bind-event-listener": "^3.0.0",
"class-variance-authority": "^0.7.1", "class-variance-authority": "^0.7.1",
"clsx": "^2.1.1", "clsx": "^2.1.1",
"cmdk": "^0.2.1", "cmdk": "^0.2.1",
"codemirror": "^6.0.1", "codemirror": "^6.0.2",
"d3": "^7.8.5", "d3": "^7.9.0",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"debounce": "^2.2.0", "debounce": "^3.0.0",
"embla-carousel-autoplay": "^8.6.0", "embla-carousel-autoplay": "^8.6.0",
"embla-carousel-react": "8.0.0-rc22", "embla-carousel-react": "8.6.0",
"flag-icons": "^7.1.0", "flag-icons": "^7.5.0",
"framer-motion": "^11.0.28", "framer-motion": "^12.38.0",
"hamburger-react": "^2.5.0", "hamburger-react": "^2.5.2",
"input-otp": "^1.2.4", "input-otp": "^1.4.2",
"javascript-time-ago": "^2.5.9", "javascript-time-ago": "^2.6.4",
"katex": "^0.16.21", "katex": "^0.16.44",
"lodash.debounce": "^4.0.8", "lodash.debounce": "^4.0.8",
"lodash.isequal": "^4.5.0", "lodash.isequal": "^4.5.0",
"lodash.throttle": "^4.1.1", "lodash.throttle": "^4.1.1",
"lottie-react": "^2.4.0", "lottie-react": "^2.4.1",
"lucide-react": "^0.476.0", "lucide-react": "^0.476.0",
"mitt": "^3.0.1", "mitt": "^3.0.1",
"nuqs": "^2.5.2", "nuqs": "^2.8.9",
"prisma-error-enum": "^0.1.3", "prisma-error-enum": "^0.1.3",
"pushmodal": "^1.0.3", "pushmodal": "^1.0.5",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"random-animal-name": "^0.1.1", "random-animal-name": "^0.1.1",
"rc-virtual-list": "^3.14.5", "rc-virtual-list": "^3.19.2",
"react": "catalog:", "react": "catalog:",
"react-animate-height": "^3.2.3", "react-animate-height": "^3.2.3",
"react-animated-numbers": "^1.1.1", "react-animated-numbers": "^1.1.1",
"react-day-picker": "^9.9.0", "react-day-picker": "^9.14.0",
"react-dom": "catalog:", "react-dom": "catalog:",
"react-grid-layout": "^1.5.2", "react-grid-layout": "^1.5.3",
"react-hook-form": "^7.50.1", "react-hook-form": "^7.72.0",
"react-in-viewport": "1.0.0-beta.8", "react-in-viewport": "1.0.0-beta.9",
"react-markdown": "^10.1.0", "react-markdown": "^10.1.0",
"react-redux": "^8.1.3", "react-redux": "^8.1.3",
"react-resizable": "^3.0.5", "react-resizable": "^3.1.3",
"react-responsive": "^9.0.2", "react-responsive": "^10.0.1",
"react-simple-maps": "3.0.0", "react-simple-maps": "3.0.0",
"react-svg-worldmap": "2.0.0-alpha.16", "react-svg-worldmap": "2.0.1",
"react-syntax-highlighter": "^15.5.0", "react-syntax-highlighter": "^16.1.1",
"react-use-websocket": "^4.7.0", "react-use-websocket": "^4.13.0",
"react-virtualized-auto-sizer": "^1.0.22", "react-virtualized-auto-sizer": "^1.0.26",
"recharts": "^2.15.4", "recharts": "^2.15.4",
"rehype-katex": "^7.0.1", "rehype-katex": "^7.0.1",
"remark-gfm": "^4.0.1", "remark-gfm": "^4.0.1",
@@ -142,42 +142,42 @@
"remark-parse": "^11.0.0", "remark-parse": "^11.0.0",
"remark-rehype": "^11.1.2", "remark-rehype": "^11.1.2",
"rrweb-player": "2.0.0-alpha.20", "rrweb-player": "2.0.0-alpha.20",
"short-unique-id": "^5.0.3", "short-unique-id": "^5.3.2",
"slugify": "^1.6.6", "slugify": "^1.6.8",
"sonner": "^1.4.0", "sonner": "^1.7.4",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^2.2.2", "superjson": "^2.2.6",
"tailwind-merge": "^3.0.2", "tailwind-merge": "^3.5.0",
"tailwindcss": "^4.0.6", "tailwindcss": "^4.2.2",
"tailwindcss-animate": "^1.0.7", "tailwindcss-animate": "^1.0.7",
"tw-animate-css": "^1.3.6", "tw-animate-css": "^1.4.0",
"usehooks-ts": "^2.14.0", "usehooks-ts": "^2.16.0",
"vite-tsconfig-paths": "^5.1.4", "vite-tsconfig-paths": "^6.1.1",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@biomejs/biome": "1.9.4", "@biomejs/biome": "2.4.10",
"@cloudflare/vite-plugin": "1.20.3", "@cloudflare/vite-plugin": "1.30.2",
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/trpc": "workspace:*", "@openpanel/trpc": "workspace:*",
"@tanstack/devtools-event-client": "^0.3.3", "@tanstack/devtools-event-client": "^0.4.3",
"@testing-library/dom": "^10.4.0", "@testing-library/dom": "^10.4.1",
"@testing-library/react": "^16.2.0", "@testing-library/react": "^16.3.2",
"@types/lodash.debounce": "^4.0.9", "@types/lodash.debounce": "^4.0.9",
"@types/lodash.isequal": "^4.5.8", "@types/lodash.isequal": "^4.5.8",
"@types/lodash.throttle": "^4.1.9", "@types/lodash.throttle": "^4.1.9",
"@types/ramda": "^0.31.0", "@types/ramda": "^0.31.1",
"@types/react": "catalog:", "@types/react": "catalog:",
"@types/react-dom": "catalog:", "@types/react-dom": "catalog:",
"@types/react-grid-layout": "^1.3.5", "@types/react-grid-layout": "^2.1.0",
"@types/react-simple-maps": "^3.0.4", "@types/react-simple-maps": "^3.0.6",
"@types/react-syntax-highlighter": "^15.5.11", "@types/react-syntax-highlighter": "^15.5.13",
"@vitejs/plugin-react": "^4.3.4", "@vitejs/plugin-react": "^4.7.0",
"jsdom": "^26.0.0", "jsdom": "^26.1.0",
"typescript": "catalog:", "typescript": "catalog:",
"vite": "^6.3.5", "vite": "^6.4.1",
"vitest": "^3.0.5", "vitest": "^3.2.4",
"web-vitals": "^4.2.4", "web-vitals": "^5.2.0",
"wrangler": "4.59.1" "wrangler": "4.78.0"
} }
} }

View File

@@ -1,61 +1,25 @@
import { TooltipComplete } from '@/components/tooltip-complete'; import { useQueryClient } from '@tanstack/react-query';
import { useDebounceState } from '@/hooks/use-debounce-state'; import { useCallback } from 'react';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import { cn } from '@/utils/cn';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef } from 'react';
import { toast } from 'sonner'; import { toast } from 'sonner';
import { AnimatedNumber } from '../animated-number'; import { AnimatedNumber } from '../animated-number';
import { TooltipComplete } from '@/components/tooltip-complete';
import { useLiveCounter } from '@/hooks/use-live-counter';
import { cn } from '@/utils/cn';
export interface LiveCounterProps { export interface LiveCounterProps {
projectId: string; projectId: string;
shareId?: string; shareId?: string;
} }
const FIFTEEN_SECONDS = 1000 * 30;
export function LiveCounter({ projectId, shareId }: LiveCounterProps) { export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
const trpc = useTRPC();
const client = useQueryClient(); const client = useQueryClient();
const counter = useDebounceState(0, 1000); const onRefresh = useCallback(() => {
const lastRefresh = useRef(Date.now()); toast('Refreshed data');
const query = useQuery( client.refetchQueries({
trpc.overview.liveVisitors.queryOptions({ type: 'active',
projectId, });
shareId, }, [client]);
}), const counter = useLiveCounter({ projectId, shareId, onRefresh });
);
useEffect(() => {
if (query.data) {
counter.set(query.data);
}
}, [query.data]);
useWS<number>(
`/live/visitors/${projectId}`,
(value) => {
if (!Number.isNaN(value)) {
counter.set(value);
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
lastRefresh.current = Date.now();
if (!document.hidden) {
toast('Refreshed data');
client.refetchQueries({
type: 'active',
});
}
}
}
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
},
);
return ( return (
<TooltipComplete <TooltipComplete
@@ -66,13 +30,13 @@ export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
<div <div
className={cn( className={cn(
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all', 'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
counter.debounced === 0 && 'bg-destructive opacity-0', counter.debounced === 0 && 'bg-destructive opacity-0'
)} )}
/> />
<div <div
className={cn( className={cn(
'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all', 'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
counter.debounced === 0 && 'bg-destructive', counter.debounced === 0 && 'bg-destructive'
)} )}
/> />
</div> </div>

View File

@@ -15,6 +15,7 @@ import {
LayoutDashboardIcon, LayoutDashboardIcon,
LayoutPanelTopIcon, LayoutPanelTopIcon,
PlusIcon, PlusIcon,
ScrollTextIcon,
SearchIcon, SearchIcon,
SparklesIcon, SparklesIcon,
TrendingUpDownIcon, TrendingUpDownIcon,
@@ -61,6 +62,7 @@ export default function SidebarProjectMenu({
<SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" /> <SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" />
<SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" /> <SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" />
<SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" /> <SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" />
<SidebarLink href={'/logs'} icon={ScrollTextIcon} label="Logs" />
<SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" /> <SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" />
<SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" /> <SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" />
<SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" /> <SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" />

View File

@@ -0,0 +1,81 @@
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { useEffect, useRef } from 'react';
import { useDebounceState } from './use-debounce-state';
import useWS from './use-ws';
import { useTRPC } from '@/integrations/trpc/react';
const FIFTEEN_SECONDS = 1000 * 15;
/** Refetch from API when WS-only updates may be stale (e.g. visitors left). */
const FALLBACK_STALE_MS = 1000 * 60;
export function useLiveCounter({
projectId,
shareId,
onRefresh,
}: {
projectId: string;
shareId?: string;
onRefresh?: () => void;
}) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const counter = useDebounceState(0, 1000);
const lastRefresh = useRef(Date.now());
const query = useQuery(
trpc.overview.liveVisitors.queryOptions({
projectId,
shareId: shareId ?? undefined,
})
);
useEffect(() => {
if (query.data) {
counter.set(query.data);
}
}, [query.data]);
useWS<number>(
`/live/visitors/${projectId}`,
(value) => {
if (!Number.isNaN(value)) {
counter.set(value);
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
lastRefresh.current = Date.now();
if (!document.hidden) {
onRefresh?.();
}
}
}
},
{
debounce: {
delay: 1000,
maxWait: 5000,
},
}
);
useEffect(() => {
const id = setInterval(async () => {
if (Date.now() - lastRefresh.current < FALLBACK_STALE_MS) {
return;
}
const data = await queryClient.fetchQuery(
trpc.overview.liveVisitors.queryOptions(
{
projectId,
shareId: shareId ?? undefined,
},
// Default query staleTime is 5m; bypass cache so this reconciliation always hits the API.
{ staleTime: 0 }
)
);
counter.set(data);
lastRefresh.current = Date.now();
}, FALLBACK_STALE_MS);
return () => clearInterval(id);
}, [projectId, shareId, trpc, queryClient, counter.set]);
return counter;
}

View File

@@ -47,6 +47,7 @@ import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/
import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references' import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references'
import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime' import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime'
import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages' import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages'
import { Route as AppOrganizationIdProjectIdLogsRouteImport } from './routes/_app.$organizationId.$projectId.logs'
import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights' import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights'
import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups' import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups'
import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards' import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards'
@@ -352,6 +353,12 @@ const AppOrganizationIdProjectIdPagesRoute =
path: '/pages', path: '/pages',
getParentRoute: () => AppOrganizationIdProjectIdRoute, getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any) } as any)
const AppOrganizationIdProjectIdLogsRoute =
AppOrganizationIdProjectIdLogsRouteImport.update({
id: '/logs',
path: '/logs',
getParentRoute: () => AppOrganizationIdProjectIdRoute,
} as any)
const AppOrganizationIdProjectIdInsightsRoute = const AppOrganizationIdProjectIdInsightsRoute =
AppOrganizationIdProjectIdInsightsRouteImport.update({ AppOrganizationIdProjectIdInsightsRouteImport.update({
id: '/insights', id: '/insights',
@@ -660,6 +667,7 @@ export interface FileRoutesByFullPath {
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute '/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute '/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute '/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute '/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -738,6 +746,7 @@ export interface FileRoutesByTo {
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute '/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute '/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute '/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute '/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -814,6 +823,7 @@ export interface FileRoutesById {
'/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute '/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
'/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute '/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
'/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute '/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
'/_app/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
'/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute '/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
'/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute '/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
'/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute '/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
@@ -905,6 +915,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/dashboards' | '/$organizationId/$projectId/dashboards'
| '/$organizationId/$projectId/groups' | '/$organizationId/$projectId/groups'
| '/$organizationId/$projectId/insights' | '/$organizationId/$projectId/insights'
| '/$organizationId/$projectId/logs'
| '/$organizationId/$projectId/pages' | '/$organizationId/$projectId/pages'
| '/$organizationId/$projectId/realtime' | '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references' | '/$organizationId/$projectId/references'
@@ -983,6 +994,7 @@ export interface FileRouteTypes {
| '/$organizationId/$projectId/dashboards' | '/$organizationId/$projectId/dashboards'
| '/$organizationId/$projectId/groups' | '/$organizationId/$projectId/groups'
| '/$organizationId/$projectId/insights' | '/$organizationId/$projectId/insights'
| '/$organizationId/$projectId/logs'
| '/$organizationId/$projectId/pages' | '/$organizationId/$projectId/pages'
| '/$organizationId/$projectId/realtime' | '/$organizationId/$projectId/realtime'
| '/$organizationId/$projectId/references' | '/$organizationId/$projectId/references'
@@ -1058,6 +1070,7 @@ export interface FileRouteTypes {
| '/_app/$organizationId/$projectId/dashboards' | '/_app/$organizationId/$projectId/dashboards'
| '/_app/$organizationId/$projectId/groups' | '/_app/$organizationId/$projectId/groups'
| '/_app/$organizationId/$projectId/insights' | '/_app/$organizationId/$projectId/insights'
| '/_app/$organizationId/$projectId/logs'
| '/_app/$organizationId/$projectId/pages' | '/_app/$organizationId/$projectId/pages'
| '/_app/$organizationId/$projectId/realtime' | '/_app/$organizationId/$projectId/realtime'
| '/_app/$organizationId/$projectId/references' | '/_app/$organizationId/$projectId/references'
@@ -1444,6 +1457,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute parentRoute: typeof AppOrganizationIdProjectIdRoute
} }
'/_app/$organizationId/$projectId/logs': {
id: '/_app/$organizationId/$projectId/logs'
path: '/logs'
fullPath: '/$organizationId/$projectId/logs'
preLoaderRoute: typeof AppOrganizationIdProjectIdLogsRouteImport
parentRoute: typeof AppOrganizationIdProjectIdRoute
}
'/_app/$organizationId/$projectId/insights': { '/_app/$organizationId/$projectId/insights': {
id: '/_app/$organizationId/$projectId/insights' id: '/_app/$organizationId/$projectId/insights'
path: '/insights' path: '/insights'
@@ -2028,6 +2048,7 @@ interface AppOrganizationIdProjectIdRouteChildren {
AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute
AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute
AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute
AppOrganizationIdProjectIdLogsRoute: typeof AppOrganizationIdProjectIdLogsRoute
AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute
AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute
AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute
@@ -2054,6 +2075,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh
AppOrganizationIdProjectIdGroupsRoute, AppOrganizationIdProjectIdGroupsRoute,
AppOrganizationIdProjectIdInsightsRoute: AppOrganizationIdProjectIdInsightsRoute:
AppOrganizationIdProjectIdInsightsRoute, AppOrganizationIdProjectIdInsightsRoute,
AppOrganizationIdProjectIdLogsRoute: AppOrganizationIdProjectIdLogsRoute,
AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute, AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute,
AppOrganizationIdProjectIdRealtimeRoute: AppOrganizationIdProjectIdRealtimeRoute:
AppOrganizationIdProjectIdRealtimeRoute, AppOrganizationIdProjectIdRealtimeRoute,

View File

@@ -0,0 +1,340 @@
import { useInfiniteQuery, useQuery } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import {
AlertCircleIcon,
AlertTriangleIcon,
BugIcon,
ChevronDownIcon,
ChevronRightIcon,
InfoIcon,
SearchIcon,
XCircleIcon,
} from 'lucide-react';
import { parseAsArrayOf, parseAsString, useQueryState } from 'nuqs';
import { useState } from 'react';
import { PageContainer } from '@/components/page-container';
import { PageHeader } from '@/components/page-header';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { useTRPC } from '@/integrations/trpc/react';
import { cn } from '@/utils/cn';
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
import type { IServiceLog } from '@openpanel/trpc';
export const Route = createFileRoute(
'/_app/$organizationId/$projectId/logs',
)({
component: Component,
head: () => ({
meta: [{ title: createProjectTitle(PAGE_TITLES.LOGS) }],
}),
});
const SEVERITY_LEVELS = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'] as const;
type SeverityLevel = (typeof SEVERITY_LEVELS)[number];
function getSeverityVariant(
severity: string,
): 'default' | 'secondary' | 'info' | 'warning' | 'destructive' | 'outline' {
switch (severity.toLowerCase()) {
case 'fatal':
case 'critical':
case 'error':
return 'destructive';
case 'warn':
case 'warning':
return 'warning';
case 'info':
return 'info';
default:
return 'outline';
}
}
function SeverityIcon({ severity }: { severity: string }) {
const s = severity.toLowerCase();
const cls = 'size-3.5 shrink-0';
if (s === 'fatal' || s === 'critical') return <XCircleIcon className={cn(cls, 'text-destructive')} />;
if (s === 'error') return <AlertCircleIcon className={cn(cls, 'text-destructive')} />;
if (s === 'warn' || s === 'warning') return <AlertTriangleIcon className={cn(cls, 'text-yellow-500')} />;
if (s === 'debug' || s === 'trace') return <BugIcon className={cn(cls, 'text-muted-foreground')} />;
return <InfoIcon className={cn(cls, 'text-blue-500')} />;
}
function LogRow({ log }: { log: IServiceLog }) {
const [expanded, setExpanded] = useState(false);
const hasDetails =
(log.attributes && Object.keys(log.attributes).length > 0) ||
(log.resource && Object.keys(log.resource).length > 0) ||
log.traceId ||
log.loggerName;
return (
<>
<button
type="button"
className={cn(
'grid w-full grid-cols-[28px_90px_80px_1fr_120px] items-center gap-3 px-4 py-2.5 text-left text-sm transition-colors hover:bg-muted/50 border-b border-border/50',
expanded && 'bg-muted/30',
)}
onClick={() => hasDetails && setExpanded((v) => !v)}
>
<span className="flex items-center justify-center text-muted-foreground">
{hasDetails ? (
expanded ? (
<ChevronDownIcon className="size-3.5" />
) : (
<ChevronRightIcon className="size-3.5" />
)
) : null}
</span>
<span className="text-muted-foreground tabular-nums text-xs">
{log.timestamp.toLocaleTimeString()}
</span>
<span>
<Badge variant={getSeverityVariant(log.severityText)} className="gap-1 font-mono uppercase text-[10px]">
<SeverityIcon severity={log.severityText} />
{log.severityText}
</Badge>
</span>
<span className="truncate font-mono text-xs">{log.body}</span>
<span className="truncate text-xs text-muted-foreground text-right">
{log.loggerName || log.os || log.device || '—'}
</span>
</button>
{expanded && (
<div className="border-b border-border/50 bg-muted/20 px-4 py-3">
<div className="grid gap-4 pl-[calc(28px+0.75rem)] text-xs sm:grid-cols-2">
{log.loggerName && (
<MetaRow label="Logger" value={log.loggerName} />
)}
{log.traceId && (
<MetaRow label="Trace ID" value={log.traceId} mono />
)}
{log.spanId && (
<MetaRow label="Span ID" value={log.spanId} mono />
)}
{log.deviceId && (
<MetaRow label="Device ID" value={log.deviceId} mono />
)}
{log.os && (
<MetaRow label="OS" value={`${log.os} ${log.osVersion}`.trim()} />
)}
{log.country && (
<MetaRow label="Location" value={[log.city, log.country].filter(Boolean).join(', ')} />
)}
{Object.keys(log.attributes).length > 0 && (
<div className="col-span-2">
<p className="mb-1.5 font-semibold text-muted-foreground">Attributes</p>
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
{Object.entries(log.attributes).map(([k, v]) => (
<div key={k} className="flex gap-2">
<span className="text-blue-500 shrink-0">{k}</span>
<span className="text-muted-foreground">=</span>
<span className="break-all">{v}</span>
</div>
))}
</div>
</div>
)}
{Object.keys(log.resource).length > 0 && (
<div className="col-span-2">
<p className="mb-1.5 font-semibold text-muted-foreground">Resource</p>
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
{Object.entries(log.resource).map(([k, v]) => (
<div key={k} className="flex gap-2">
<span className="text-emerald-600 shrink-0">{k}</span>
<span className="text-muted-foreground">=</span>
<span className="break-all">{v}</span>
</div>
))}
</div>
</div>
)}
</div>
</div>
)}
</>
);
}
function MetaRow({ label, value, mono }: { label: string; value: string; mono?: boolean }) {
return (
<div className="flex gap-2">
<span className="w-24 shrink-0 font-medium text-muted-foreground">{label}</span>
<span className={cn('break-all', mono && 'font-mono')}>{value}</span>
</div>
);
}
function Component() {
const { projectId } = Route.useParams();
const trpc = useTRPC();
const [search, setSearch] = useQueryState('search', parseAsString.withDefault(''));
const [severities, setSeverities] = useQueryState(
'severity',
parseAsArrayOf(parseAsString).withDefault([]),
);
const countsQuery = useQuery(
trpc.log.severityCounts.queryOptions({ projectId }),
);
const logsQuery = useInfiniteQuery(
trpc.log.list.infiniteQueryOptions(
{
projectId,
search: search || undefined,
severity: severities.length > 0 ? (severities as SeverityLevel[]) : undefined,
take: 50,
},
{
getNextPageParam: (lastPage) => lastPage.meta.next,
},
),
);
const logs = logsQuery.data?.pages.flatMap((p) => p.data) ?? [];
const counts = countsQuery.data ?? {};
const toggleSeverity = (s: string) => {
setSeverities((prev) =>
prev.includes(s) ? prev.filter((x) => x !== s) : [...prev, s],
);
};
return (
<PageContainer>
<PageHeader
className="mb-6"
title="Logs"
description="Captured device and application logs in OpenTelemetry format"
/>
{/* Severity filter chips */}
<div className="mb-4 flex flex-wrap items-center gap-2">
{SEVERITY_LEVELS.map((s) => {
const active = severities.includes(s);
const count = counts[s] ?? 0;
return (
<button
key={s}
type="button"
onClick={() => toggleSeverity(s)}
className={cn(
'inline-flex items-center gap-1.5 rounded-full border px-3 py-1 text-xs font-medium transition-colors',
active
? 'border-foreground bg-foreground text-background'
: 'border-border bg-transparent text-muted-foreground hover:border-foreground/50 hover:text-foreground',
)}
>
<SeverityIcon severity={s} />
<span className="uppercase">{s}</span>
{count > 0 && (
<span className={cn('tabular-nums', active ? 'opacity-70' : 'opacity-50')}>
{count.toLocaleString()}
</span>
)}
</button>
);
})}
{severities.length > 0 && (
<button
type="button"
onClick={() => setSeverities([])}
className="text-xs text-muted-foreground hover:text-foreground underline"
>
Clear
</button>
)}
</div>
{/* Search */}
<div className="mb-4 relative">
<SearchIcon className="absolute left-3 top-1/2 -translate-y-1/2 size-4 text-muted-foreground" />
<Input
className="pl-9"
placeholder="Search log messages…"
value={search}
onChange={(e) => setSearch(e.target.value || null)}
/>
</div>
{/* Log table */}
<div className="rounded-lg border bg-card overflow-hidden">
{/* Header */}
<div className="grid grid-cols-[28px_90px_80px_1fr_120px] gap-3 border-b bg-muted/50 px-4 py-2 text-xs font-medium text-muted-foreground">
<span />
<span>Time</span>
<span>Level</span>
<span>Message</span>
<span className="text-right">Source</span>
</div>
{logsQuery.isPending && (
<div className="space-y-0 divide-y">
{Array.from({ length: 10 }).map((_, i) => (
<div key={i} className="h-10 animate-pulse bg-muted/30" />
))}
</div>
)}
{!logsQuery.isPending && logs.length === 0 && (
<div className="flex flex-col items-center gap-2 py-16 text-center text-muted-foreground">
<ScrollTextIcon className="size-10 opacity-30" />
<p className="text-sm font-medium">No logs found</p>
<p className="text-xs">
{search || severities.length > 0
? 'Try adjusting your filters'
: 'Logs will appear here once your app starts sending them'}
</p>
</div>
)}
{logs.map((log) => (
<LogRow key={log.id} log={log} />
))}
{logsQuery.hasNextPage && (
<div className="flex justify-center p-4">
<Button
variant="outline"
size="sm"
onClick={() => logsQuery.fetchNextPage()}
disabled={logsQuery.isFetchingNextPage}
>
{logsQuery.isFetchingNextPage ? 'Loading…' : 'Load more'}
</Button>
</div>
)}
</div>
</PageContainer>
);
}
function ScrollTextIcon({ className }: { className?: string }) {
return (
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
className={className}
>
<path d="M8 21h12a2 2 0 0 0 2-2v-2H10v2a2 2 0 1 1-4 0V5a2 2 0 1 0-4 0v3h4" />
<path d="M19 17V5a2 2 0 0 0-2-2H4" />
<path d="M15 8h-5" />
<path d="M15 12h-5" />
</svg>
);
}

View File

@@ -1,12 +1,11 @@
import { AnimatedNumber } from '@/components/animated-number';
import { Ping } from '@/components/ping';
import { useNumber } from '@/hooks/use-numer-formatter';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import type { RouterOutputs } from '@/trpc/client';
import { useQuery, useQueryClient } from '@tanstack/react-query'; import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router'; import { createFileRoute } from '@tanstack/react-router';
import { z } from 'zod'; import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number';
import { Ping } from '@/components/ping';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import type { RouterOutputs } from '@/trpc/client';
const widgetSearchSchema = z.object({ const widgetSearchSchema = z.object({
shareId: z.string(), shareId: z.string(),
@@ -20,33 +19,33 @@ export const Route = createFileRoute('/widget/counter')({
}); });
function RouteComponent() { function RouteComponent() {
const { shareId, limit, color } = Route.useSearch(); const { shareId } = Route.useSearch();
const trpc = useTRPC(); const trpc = useTRPC();
// Fetch widget data // Fetch widget data
const { data, isLoading } = useQuery( const { data, isLoading } = useQuery(
trpc.widget.counter.queryOptions({ shareId }), trpc.widget.counter.queryOptions({ shareId })
); );
if (isLoading) { if (isLoading) {
return ( return (
<div className="flex items-center gap-2 px-2 h-8"> <div className="flex h-8 items-center gap-2 px-2">
<Ping /> <Ping />
<AnimatedNumber value={0} suffix=" unique visitors" /> <AnimatedNumber suffix=" unique visitors" value={0} />
</div> </div>
); );
} }
if (!data) { if (!data) {
return ( return (
<div className="flex items-center gap-2 px-2 h-8"> <div className="flex h-8 items-center gap-2 px-2">
<Ping className="bg-orange-500" /> <Ping className="bg-orange-500" />
<AnimatedNumber value={0} suffix=" unique visitors" /> <AnimatedNumber suffix=" unique visitors" value={0} />
</div> </div>
); );
} }
return <CounterWidget shareId={shareId} data={data} />; return <CounterWidget data={data} shareId={shareId} />;
} }
interface RealtimeWidgetProps { interface RealtimeWidgetProps {
@@ -57,30 +56,29 @@ interface RealtimeWidgetProps {
function CounterWidget({ shareId, data }: RealtimeWidgetProps) { function CounterWidget({ shareId, data }: RealtimeWidgetProps) {
const trpc = useTRPC(); const trpc = useTRPC();
const queryClient = useQueryClient(); const queryClient = useQueryClient();
const number = useNumber();
// WebSocket subscription for real-time updates // WebSocket subscription for real-time updates
useWS<number>( useWS<number>(
`/live/visitors/${data.projectId}`, `/live/visitors/${data.projectId}`,
(res) => { () => {
if (!document.hidden) { if (!document.hidden) {
queryClient.refetchQueries( queryClient.refetchQueries(
trpc.widget.counter.queryFilter({ shareId }), trpc.widget.counter.queryFilter({ shareId })
); );
} }
}, },
{ {
debounce: { debounce: {
delay: 1000, delay: 1000,
maxWait: 60000, maxWait: 60_000,
}, },
}, }
); );
return ( return (
<div className="flex items-center gap-2 px-2 h-8"> <div className="flex h-8 items-center gap-2 px-2">
<Ping /> <Ping />
<AnimatedNumber value={data.counter} suffix=" unique visitors" /> <AnimatedNumber suffix=" unique visitors" value={data.counter} />
</div> </div>
); );
} }

View File

@@ -1,3 +1,15 @@
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import type React from 'react';
import {
Bar,
BarChart,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number'; import { AnimatedNumber } from '@/components/animated-number';
import { import {
ChartTooltipContainer, ChartTooltipContainer,
@@ -14,18 +26,6 @@ import { countries } from '@/translations/countries';
import type { RouterOutputs } from '@/trpc/client'; import type { RouterOutputs } from '@/trpc/client';
import { cn } from '@/utils/cn'; import { cn } from '@/utils/cn';
import { getChartColor } from '@/utils/theme'; import { getChartColor } from '@/utils/theme';
import { useQuery, useQueryClient } from '@tanstack/react-query';
import { createFileRoute } from '@tanstack/react-router';
import type React from 'react';
import {
Bar,
BarChart,
ResponsiveContainer,
Tooltip,
XAxis,
YAxis,
} from 'recharts';
import { z } from 'zod';
const widgetSearchSchema = z.object({ const widgetSearchSchema = z.object({
shareId: z.string(), shareId: z.string(),
@@ -44,7 +44,7 @@ function RouteComponent() {
// Fetch widget data // Fetch widget data
const { data: widgetData, isLoading } = useQuery( const { data: widgetData, isLoading } = useQuery(
trpc.widget.realtimeData.queryOptions({ shareId }), trpc.widget.realtimeData.queryOptions({ shareId })
); );
if (isLoading) { if (isLoading) {
@@ -53,10 +53,10 @@ function RouteComponent() {
if (!widgetData) { if (!widgetData) {
return ( return (
<div className="flex h-screen w-full center-center bg-background text-foreground col p-4"> <div className="center-center col flex h-screen w-full bg-background p-4 text-foreground">
<LogoSquare className="size-10 mb-4" /> <LogoSquare className="mb-4 size-10" />
<h1 className="text-xl font-semibold">Widget not found</h1> <h1 className="font-semibold text-xl">Widget not found</h1>
<p className="mt-2 text-sm text-muted-foreground"> <p className="mt-2 text-muted-foreground text-sm">
This widget is not available or has been removed. This widget is not available or has been removed.
</p> </p>
</div> </div>
@@ -65,10 +65,10 @@ function RouteComponent() {
return ( return (
<RealtimeWidget <RealtimeWidget
shareId={shareId}
limit={limit}
data={widgetData}
color={color} color={color}
data={widgetData}
limit={limit}
shareId={shareId}
/> />
); );
} }
@@ -83,7 +83,6 @@ interface RealtimeWidgetProps {
function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) { function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const trpc = useTRPC(); const trpc = useTRPC();
const queryClient = useQueryClient(); const queryClient = useQueryClient();
const number = useNumber();
// WebSocket subscription for real-time updates // WebSocket subscription for real-time updates
useWS<number>( useWS<number>(
@@ -91,16 +90,16 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
() => { () => {
if (!document.hidden) { if (!document.hidden) {
queryClient.refetchQueries( queryClient.refetchQueries(
trpc.widget.realtimeData.queryFilter({ shareId }), trpc.widget.realtimeData.queryFilter({ shareId })
); );
} }
}, },
{ {
debounce: { debounce: {
delay: 1000, delay: 1000,
maxWait: 60000, maxWait: 60_000,
}, },
}, }
); );
const maxDomain = const maxDomain =
@@ -111,8 +110,12 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const referrers = data.referrers.length > 0 ? 1 : 0; const referrers = data.referrers.length > 0 ? 1 : 0;
const paths = data.paths.length > 0 ? 1 : 0; const paths = data.paths.length > 0 ? 1 : 0;
const value = countries + referrers + paths; const value = countries + referrers + paths;
if (value === 3) return 'md:grid-cols-3'; if (value === 3) {
if (value === 2) return 'md:grid-cols-2'; return 'md:grid-cols-3';
}
if (value === 2) {
return 'md:grid-cols-2';
}
return 'md:grid-cols-1'; return 'md:grid-cols-1';
})(); })();
@@ -120,10 +123,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
<div className="flex h-screen w-full flex-col bg-background text-foreground"> <div className="flex h-screen w-full flex-col bg-background text-foreground">
{/* Header with live counter */} {/* Header with live counter */}
<div className="p-6 pb-3"> <div className="p-6 pb-3">
<div className="flex items-center justify-between w-full h-4"> <div className="flex h-4 w-full items-center justify-between">
<div className="flex items-center gap-3 w-full"> <div className="flex w-full items-center gap-3">
<Ping /> <Ping />
<div className="text-sm font-medium text-muted-foreground flex-1"> <div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES USERS IN LAST 30 MINUTES
</div> </div>
{data.project.domain && <SerieIcon name={data.project.domain} />} {data.project.domain && <SerieIcon name={data.project.domain} />}
@@ -131,14 +134,14 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
</div> </div>
<div className="row"> <div className="row">
<div className="font-mono text-6xl font-bold h-18 text-foreground"> <div className="h-18 font-bold font-mono text-6xl text-foreground">
<AnimatedNumber value={data.liveCount} /> <AnimatedNumber value={data.liveCount} />
</div> </div>
</div> </div>
<div className="flex h-20 w-full flex-col -mt-4"> <div className="-mt-4 flex h-20 w-full flex-col">
<div className="flex-1"> <div className="flex-1">
<ResponsiveContainer width="100%" height="100%"> <ResponsiveContainer height="100%" width="100%">
<BarChart <BarChart
data={data.histogram} data={data.histogram}
margin={{ top: 0, right: 0, left: 0, bottom: 0 }} margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
@@ -148,22 +151,22 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
cursor={{ fill: 'var(--def-100)', radius: 4 }} cursor={{ fill: 'var(--def-100)', radius: 4 }}
/> />
<XAxis <XAxis
dataKey="time"
axisLine={false} axisLine={false}
tickLine={false} dataKey="time"
interval="preserveStartEnd"
tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }} tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }}
tickLine={false}
ticks={[ ticks={[
data.histogram[0].time, data.histogram[0].time,
data.histogram[data.histogram.length - 1].time, data.histogram[data.histogram.length - 1].time,
]} ]}
interval="preserveStartEnd"
/> />
<YAxis hide domain={[0, maxDomain]} /> <YAxis domain={[0, maxDomain]} hide />
<Bar <Bar
dataKey="sessionCount" dataKey="sessionCount"
fill={color || 'var(--chart-0)'}
isAnimationActive={false} isAnimationActive={false}
radius={[4, 4, 4, 4]} radius={[4, 4, 4, 4]}
fill={color || 'var(--chart-0)'}
/> />
</BarChart> </BarChart>
</ResponsiveContainer> </ResponsiveContainer>
@@ -174,24 +177,24 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{(data.countries.length > 0 || {(data.countries.length > 0 ||
data.referrers.length > 0 || data.referrers.length > 0 ||
data.paths.length > 0) && ( data.paths.length > 0) && (
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar border-t"> <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto border-t p-6">
<div className={cn('grid grid-cols-1 gap-6', grids)}> <div className={cn('grid grid-cols-1 gap-6', grids)}>
{/* Countries */} {/* Countries */}
{data.countries.length > 0 && ( {data.countries.length > 0 && (
<div className="flex flex-col"> <div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground"> <div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY COUNTRY
</div> </div>
<div className="col"> <div className="col">
{(() => { {(() => {
const { visible, rest, restCount } = getRestItems( const { visible, rest, restCount } = getRestItems(
data.countries, data.countries,
limit, limit
); );
return ( return (
<> <>
{visible.map((item) => ( {visible.map((item) => (
<RowItem key={item.country} count={item.count}> <RowItem count={item.count} key={item.country}>
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<SerieIcon name={item.country} /> <SerieIcon name={item.country} />
<span className="text-sm"> <span className="text-sm">
@@ -224,19 +227,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Referrers */} {/* Referrers */}
{data.referrers.length > 0 && ( {data.referrers.length > 0 && (
<div className="flex flex-col"> <div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground"> <div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER REFERRER
</div> </div>
<div className="col"> <div className="col">
{(() => { {(() => {
const { visible, rest, restCount } = getRestItems( const { visible, rest, restCount } = getRestItems(
data.referrers, data.referrers,
limit, limit
); );
return ( return (
<> <>
{visible.map((item) => ( {visible.map((item) => (
<RowItem key={item.referrer} count={item.count}> <RowItem count={item.count} key={item.referrer}>
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<SerieIcon name={item.referrer} /> <SerieIcon name={item.referrer} />
<span className="truncate text-sm"> <span className="truncate text-sm">
@@ -263,19 +266,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Paths */} {/* Paths */}
{data.paths.length > 0 && ( {data.paths.length > 0 && (
<div className="flex flex-col"> <div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground"> <div className="mb-3 font-medium text-muted-foreground text-xs">
PATH PATH
</div> </div>
<div className="col"> <div className="col">
{(() => { {(() => {
const { visible, rest, restCount } = getRestItems( const { visible, rest, restCount } = getRestItems(
data.paths, data.paths,
limit, limit
); );
return ( return (
<> <>
{visible.map((item) => ( {visible.map((item) => (
<RowItem key={item.path} count={item.count}> <RowItem count={item.count} key={item.path}>
<span className="truncate text-sm"> <span className="truncate text-sm">
{item.path} {item.path}
</span> </span>
@@ -303,10 +306,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
} }
// Custom tooltip component that uses portals to escape overflow hidden // Custom tooltip component that uses portals to escape overflow hidden
const CustomTooltip = ({ active, payload, coordinate }: any) => { const CustomTooltip = ({ active, payload }: any) => {
const number = useNumber(); const number = useNumber();
if (!active || !payload || !payload.length) { if (!(active && payload && payload.length)) {
return null; return null;
} }
@@ -328,10 +331,13 @@ const CustomTooltip = ({ active, payload, coordinate }: any) => {
function RowItem({ function RowItem({
children, children,
count, count,
}: { children: React.ReactNode; count: number }) { }: {
children: React.ReactNode;
count: number;
}) {
const number = useNumber(); const number = useNumber();
return ( return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3"> <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
{children} {children}
<span className="font-semibold">{number.short(count)}</span> <span className="font-semibold">{number.short(count)}</span>
</div> </div>
@@ -340,7 +346,7 @@ function RowItem({
function getRestItems<T extends { count: number }>( function getRestItems<T extends { count: number }>(
items: T[], items: T[],
limit: number, limit: number
): { visible: T[]; rest: T[]; restCount: number } { ): { visible: T[]; rest: T[]; restCount: number } {
const visible = items.slice(0, limit); const visible = items.slice(0, limit);
const rest = items.slice(limit); const rest = items.slice(limit);
@@ -375,7 +381,7 @@ function RestRow({
: 'paths'; : 'paths';
return ( return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3"> <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
<span className="truncate"> <span className="truncate">
{firstName} and {otherCount} more {typeLabel}... {firstName} and {otherCount} more {typeLabel}...
</span> </span>
@@ -434,13 +440,13 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
const itemCount = Math.min(limit, 5); const itemCount = Math.min(limit, 5);
return ( return (
<div className="flex h-screen w-full flex-col bg-background text-foreground animate-pulse"> <div className="flex h-screen w-full animate-pulse flex-col bg-background text-foreground">
{/* Header with live counter */} {/* Header with live counter */}
<div className="border-b p-6 pb-3"> <div className="border-b p-6 pb-3">
<div className="flex items-center justify-between w-full h-4"> <div className="flex h-4 w-full items-center justify-between">
<div className="flex items-center gap-3 w-full"> <div className="flex w-full items-center gap-3">
<div className="size-2 rounded-full bg-muted" /> <div className="size-2 rounded-full bg-muted" />
<div className="text-sm font-medium text-muted-foreground flex-1"> <div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES USERS IN LAST 30 MINUTES
</div> </div>
</div> </div>
@@ -448,35 +454,35 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
</div> </div>
<div className="row"> <div className="row">
<div className="font-mono text-6xl font-bold h-18 flex items-center py-4 gap-1 row"> <div className="row flex h-18 items-center gap-1 py-4 font-bold font-mono text-6xl">
<div className="h-full w-6 bg-muted rounded" /> <div className="h-full w-6 rounded bg-muted" />
<div className="h-full w-6 bg-muted rounded" /> <div className="h-full w-6 rounded bg-muted" />
</div> </div>
</div> </div>
<div className="flex h-20 w-full flex-col -mt-4 pb-2.5"> <div className="-mt-4 flex h-20 w-full flex-col pb-2.5">
<div className="flex-1 row gap-1 h-full"> <div className="row h-full flex-1 gap-1">
{SKELETON_HISTOGRAM.map((item, index) => ( {SKELETON_HISTOGRAM.map((item, index) => (
<div <div
className="mt-auto h-full w-full rounded bg-muted"
key={index.toString()} key={index.toString()}
style={{ height: `${item}%` }} style={{ height: `${item}%` }}
className="h-full w-full bg-muted rounded mt-auto"
/> />
))} ))}
</div> </div>
<div className="row justify-between pt-2"> <div className="row justify-between pt-2">
<div className="h-3 w-8 bg-muted rounded" /> <div className="h-3 w-8 rounded bg-muted" />
<div className="h-3 w-8 bg-muted rounded" /> <div className="h-3 w-8 rounded bg-muted" />
</div> </div>
</div> </div>
</div> </div>
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar"> <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto p-6">
{/* Countries, Referrers, and Paths skeleton */} {/* Countries, Referrers, and Paths skeleton */}
<div className="grid grid-cols-1 gap-6 md:grid-cols-3"> <div className="grid grid-cols-1 gap-6 md:grid-cols-3">
{/* Countries skeleton */} {/* Countries skeleton */}
<div className="flex flex-col"> <div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground"> <div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY COUNTRY
</div> </div>
<div className="col"> <div className="col">
@@ -488,7 +494,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Referrers skeleton */} {/* Referrers skeleton */}
<div className="flex flex-col"> <div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground"> <div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER REFERRER
</div> </div>
<div className="col"> <div className="col">
@@ -500,7 +506,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Paths skeleton */} {/* Paths skeleton */}
<div className="flex flex-col"> <div className="flex flex-col">
<div className="mb-3 text-xs font-medium text-muted-foreground"> <div className="mb-3 font-medium text-muted-foreground text-xs">
PATH PATH
</div> </div>
<div className="col"> <div className="col">
@@ -517,12 +523,12 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
function RowItemSkeleton() { function RowItemSkeleton() {
return ( return (
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b -mx-3"> <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm">
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<div className="size-5 rounded bg-muted" /> <div className="size-5 rounded bg-muted" />
<div className="h-4 w-24 bg-muted rounded" /> <div className="h-4 w-24 rounded bg-muted" />
</div> </div>
<div className="h-4 w-8 bg-muted rounded" /> <div className="h-4 w-8 rounded bg-muted" />
</div> </div>
); );
} }

View File

@@ -88,6 +88,7 @@ export const PAGE_TITLES = {
MEMBERS: 'Members', MEMBERS: 'Members',
BILLING: 'Billing', BILLING: 'Billing',
CHAT: 'AI Assistant', CHAT: 'AI Assistant',
LOGS: 'Logs',
REALTIME: 'Realtime', REALTIME: 'Realtime',
REFERENCES: 'References', REFERENCES: 'References',
INSIGHTS: 'Insights', INSIGHTS: 'Insights',

View File

@@ -10,15 +10,15 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/web": "workspace:*", "@openpanel/web": "workspace:*",
"react": "^19.0.0", "react": "^19.2.4",
"react-dom": "^19.0.0", "react-dom": "^19.2.4",
"react-router-dom": "^7.13.1" "react-router-dom": "^7.13.2"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "^19.0.0", "@types/react": "^19.2.14",
"@types/react-dom": "^19.0.0", "@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^4.3.4", "@vitejs/plugin-react": "^4.7.0",
"typescript": "catalog:", "typescript": "catalog:",
"vite": "^6.0.0" "vite": "^6.4.1"
} }
} }

View File

@@ -1,4 +1,4 @@
ARG NODE_VERSION=22.20.0 ARG NODE_VERSION=22.22.2
FROM node:${NODE_VERSION}-slim AS base FROM node:${NODE_VERSION}-slim AS base

View File

@@ -11,8 +11,8 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@bull-board/api": "6.14.0", "@bull-board/api": "6.20.6",
"@bull-board/express": "6.14.0", "@bull-board/express": "6.20.6",
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*", "@openpanel/email": "workspace:*",
@@ -24,24 +24,25 @@
"@openpanel/payments": "workspace:*", "@openpanel/payments": "workspace:*",
"@openpanel/queue": "workspace:*", "@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"bullmq": "^5.63.0", "@openpanel/validation": "workspace:*",
"date-fns": "^3.3.1", "bullmq": "^5.71.1",
"express": "^4.18.2", "date-fns": "^3.6.0",
"express": "^4.22.1",
"groupmq": "catalog:", "groupmq": "catalog:",
"prom-client": "^15.1.3", "prom-client": "^15.1.3",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"source-map-support": "^0.5.21", "source-map-support": "^0.5.21",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"uuid": "^9.0.1" "uuid": "^9.0.1"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/express": "^4.17.21", "@types/express": "^5.0.6",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/source-map-support": "^0.5.10", "@types/source-map-support": "^0.5.10",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"@types/uuid": "^9.0.8", "@types/uuid": "^11.0.0",
"tsdown": "0.14.2", "tsdown": "0.21.7",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -73,6 +73,11 @@ export async function bootCron() {
type: 'flushGroups', type: 'flushGroups',
pattern: 1000 * 10, pattern: 1000 * 10,
}, },
{
name: 'flush',
type: 'flushLogs',
pattern: 1000 * 10,
},
{ {
name: 'insightsDaily', name: 'insightsDaily',
type: 'insightsDaily', type: 'insightsDaily',

View File

@@ -8,6 +8,7 @@ import {
gscQueue, gscQueue,
importQueue, importQueue,
insightsQueue, insightsQueue,
logsQueue,
miscQueue, miscQueue,
notificationQueue, notificationQueue,
queueLogger, queueLogger,
@@ -22,6 +23,7 @@ import { incomingEvent } from './jobs/events.incoming-event';
import { gscJob } from './jobs/gsc'; import { gscJob } from './jobs/gsc';
import { importJob } from './jobs/import'; import { importJob } from './jobs/import';
import { insightsProjectJob } from './jobs/insights'; import { insightsProjectJob } from './jobs/insights';
import { incomingLog } from './jobs/logs.incoming-log';
import { miscJob } from './jobs/misc'; import { miscJob } from './jobs/misc';
import { notificationJob } from './jobs/notification'; import { notificationJob } from './jobs/notification';
import { sessionsJob } from './jobs/sessions'; import { sessionsJob } from './jobs/sessions';
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
'import', 'import',
'insights', 'insights',
'gsc', 'gsc',
'logs',
]; ];
} }
@@ -221,6 +224,20 @@ export function bootWorkers() {
logger.info('Started worker for gsc', { concurrency }); logger.info('Started worker for gsc', { concurrency });
} }
// Start logs worker
if (enabledQueues.includes('logs')) {
const concurrency = getConcurrencyFor('logs', 10);
const logsWorker = new Worker(
logsQueue.name,
async (job) => {
await incomingLog(job.data.payload);
},
{ ...workerOptions, concurrency },
);
workers.push(logsWorker);
logger.info('Started worker for logs', { concurrency });
}
if (workers.length === 0) { if (workers.length === 0) {
logger.warn( logger.warn(
'No workers started. Check ENABLED_QUEUES environment variable.' 'No workers started. Check ENABLED_QUEUES environment variable.'

View File

@@ -1,6 +1,6 @@
import type { Job } from 'bullmq'; import type { Job } from 'bullmq';
import { eventBuffer, groupBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db'; import { eventBuffer, groupBuffer, logBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
import type { CronQueuePayload } from '@openpanel/queue'; import type { CronQueuePayload } from '@openpanel/queue';
import { jobdeleteProjects } from './cron.delete-projects'; import { jobdeleteProjects } from './cron.delete-projects';
@@ -33,6 +33,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
case 'flushGroups': { case 'flushGroups': {
return await groupBuffer.tryFlush(); return await groupBuffer.tryFlush();
} }
case 'flushLogs': {
return await logBuffer.tryFlush();
}
case 'ping': { case 'ping': {
return await ping(); return await ping();
} }

View File

@@ -0,0 +1,63 @@
import type { IClickhouseLog } from '@openpanel/db';
import { logBuffer } from '@openpanel/db';
import type { LogsQueuePayload } from '@openpanel/queue';
import { SEVERITY_TEXT_TO_NUMBER } from '@openpanel/validation';
import { logger as baseLogger } from '@/utils/logger';
export async function incomingLog(
payload: LogsQueuePayload['payload'],
): Promise<void> {
const logger = baseLogger.child({ projectId: payload.projectId });
try {
const { log, uaInfo, geo, deviceId, sessionId, projectId, headers } = payload;
const sdkName = headers['openpanel-sdk-name'] ?? '';
const sdkVersion = headers['openpanel-sdk-version'] ?? '';
const severityNumber =
log.severityNumber ??
SEVERITY_TEXT_TO_NUMBER[log.severity] ??
9; // INFO fallback
const row: IClickhouseLog = {
project_id: projectId,
device_id: deviceId,
profile_id: log.profileId ? String(log.profileId) : '',
session_id: sessionId,
timestamp: log.timestamp,
observed_at: new Date().toISOString(),
severity_number: severityNumber,
severity_text: log.severity,
body: log.body,
trace_id: log.traceId ?? '',
span_id: log.spanId ?? '',
trace_flags: log.traceFlags ?? 0,
logger_name: log.loggerName ?? '',
attributes: log.attributes ?? {},
resource: log.resource ?? {},
sdk_name: sdkName,
sdk_version: sdkVersion,
country: geo.country ?? '',
city: geo.city ?? '',
region: geo.region ?? '',
os: uaInfo.os ?? '',
os_version: uaInfo.osVersion ?? '',
browser: uaInfo.isServer ? '' : (uaInfo.browser ?? ''),
browser_version: uaInfo.isServer ? '' : (uaInfo.browserVersion ?? ''),
device: uaInfo.device ?? '',
brand: uaInfo.isServer ? '' : (uaInfo.brand ?? ''),
model: uaInfo.isServer ? '' : (uaInfo.model ?? ''),
};
logBuffer.add(row);
logger.info('Log queued', {
severity: log.severity,
loggerName: log.loggerName,
});
} catch (error) {
logger.error('Failed to process incoming log', { error });
throw error;
}
}

146
docker-compose.prod.yml Normal file
View File

@@ -0,0 +1,146 @@
services:
op-db:
image: postgres:18.3-alpine
restart: always
volumes:
- op-db-data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
op-kv:
image: redis:8.6.2-alpine
restart: always
volumes:
- op-kv-data:/data
command: ["redis-server", "--maxmemory-policy", "noeviction"]
healthcheck:
test: ["CMD-SHELL", "redis-cli ping"]
interval: 10s
timeout: 5s
retries: 5
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
op-ch:
image: clickhouse/clickhouse-server:26.3.2.3
restart: always
environment:
- CLICKHOUSE_DEFAULT_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse}
volumes:
- op-ch-data:/var/lib/clickhouse
- op-ch-logs:/var/log/clickhouse-server
- ./self-hosting/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/op-config.xml:ro
- ./self-hosting/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro
- ./self-hosting/clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
healthcheck:
test: ["CMD-SHELL", "clickhouse-client --query 'SELECT 1'"]
interval: 10s
timeout: 5s
retries: 5
ulimits:
nofile:
soft: 262144
hard: 262144
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
op-api:
image: git.zias.be/zias/openpanel-api:latest
restart: always
ports:
- "3001:3000"
command: >
sh -c "
echo 'Running migrations...'
CI=true pnpm -r run migrate:deploy
pnpm start
"
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
depends_on:
op-db:
condition: service_healthy
op-ch:
condition: service_healthy
op-kv:
condition: service_healthy
env_file:
- .env.prod
logging:
driver: "json-file"
options:
max-size: "50m"
max-file: "3"
op-dashboard:
image: git.zias.be/zias/openpanel-dashboard:latest
restart: always
ports:
- "3000:3000"
depends_on:
op-api:
condition: service_healthy
env_file:
- .env.prod
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
logging:
driver: "json-file"
options:
max-size: "20m"
max-file: "3"
op-worker:
image: git.zias.be/zias/openpanel-worker:latest
restart: always
ports:
- "3002:3000"
depends_on:
op-api:
condition: service_healthy
env_file:
- .env.prod
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
interval: 10s
timeout: 5s
retries: 5
logging:
driver: "json-file"
options:
max-size: "30m"
max-file: "3"
volumes:
op-db-data:
driver: local
op-kv-data:
driver: local
op-ch-data:
driver: local
op-ch-logs:
driver: local

View File

@@ -2,7 +2,7 @@ version: "3"
services: services:
op-db: op-db:
image: postgres:14-alpine image: postgres:18.3-alpine
restart: always restart: always
volumes: volumes:
- ./docker/data/op-db-data:/var/lib/postgresql/data - ./docker/data/op-db-data:/var/lib/postgresql/data
@@ -13,7 +13,7 @@ services:
- POSTGRES_PASSWORD=postgres - POSTGRES_PASSWORD=postgres
op-kv: op-kv:
image: redis:7.2.5-alpine image: redis:8.6.2-alpine
restart: always restart: always
volumes: volumes:
- ./docker/data/op-kv-data:/data - ./docker/data/op-kv-data:/data
@@ -22,7 +22,7 @@ services:
- 6379:6379 - 6379:6379
op-ch: op-ch:
image: clickhouse/clickhouse-server:26.1.3.52 image: clickhouse/clickhouse-server:26.3.2.3
restart: always restart: always
volumes: volumes:
- ./docker/data/op-ch-data:/var/lib/clickhouse - ./docker/data/op-ch-data:/var/lib/clickhouse

View File

@@ -5,7 +5,7 @@
"private": true, "private": true,
"license": "MIT", "license": "MIT",
"author": "Carl-Gerhard Lindesvärd", "author": "Carl-Gerhard Lindesvärd",
"packageManager": "pnpm@10.6.2", "packageManager": "pnpm@10.33.0",
"scripts": { "scripts": {
"test": "vitest run", "test": "vitest run",
"gen:bots": "pnpm -r --filter api gen:bots", "gen:bots": "pnpm -r --filter api gen:bots",
@@ -30,11 +30,11 @@
"pre-push": "[ -n \"$SKIP_HOOKS\" ] || (pnpm typecheck && pnpm test)" "pre-push": "[ -n \"$SKIP_HOOKS\" ] || (pnpm typecheck && pnpm test)"
}, },
"dependencies": { "dependencies": {
"@hyperdx/node-opentelemetry": "^0.8.1", "@hyperdx/node-opentelemetry": "^0.10.3",
"dotenv-cli": "^7.3.0", "dotenv-cli": "^7.4.4",
"semver": "^7.5.4", "semver": "^7.7.4",
"typescript": "catalog:", "typescript": "catalog:",
"winston": "^3.14.2" "winston": "^3.19.0"
}, },
"trustedDependencies": [ "trustedDependencies": [
"@biomejs/biome", "@biomejs/biome",
@@ -49,15 +49,16 @@
"sharp" "sharp"
], ],
"devDependencies": { "devDependencies": {
"@biomejs/biome": "2.3.15", "@biomejs/biome": "2.4.10",
"depcheck": "^1.4.7", "depcheck": "^1.4.7",
"simple-git-hooks": "^2.12.1", "simple-git-hooks": "^2.13.1",
"ultracite": "7.2.0", "ultracite": "7.4.0",
"vitest": "^3.0.4" "vitest": "^3.2.4"
}, },
"pnpm": { "pnpm": {
"patchedDependencies": { "patchedDependencies": {
"nuqs": "patches/nuqs.patch" "nuqs": "patches/nuqs.patch",
"buffer-equal-constant-time": "patches/buffer-equal-constant-time.patch"
}, },
"overrides": { "overrides": {
"rolldown": "1.0.0-beta.43", "rolldown": "1.0.0-beta.43",

View File

@@ -12,13 +12,13 @@
"@openpanel/validation": "workspace:^", "@openpanel/validation": "workspace:^",
"@oslojs/crypto": "^1.0.1", "@oslojs/crypto": "^1.0.1",
"@oslojs/encoding": "^1.1.0", "@oslojs/encoding": "^1.1.0",
"arctic": "^2.3.0" "arctic": "^2.3.4"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/react": "catalog:", "@types/react": "catalog:",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
}, },
"peerDependencies": { "peerDependencies": {

View File

@@ -15,15 +15,15 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/constants": "workspace:*", "@openpanel/constants": "workspace:*",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"lru-cache": "^11.2.4", "lru-cache": "^11.2.7",
"luxon": "^3.7.2", "luxon": "^3.7.2",
"mathjs": "^12.3.2", "mathjs": "^15.1.1",
"nanoid": "^5.1.6", "nanoid": "^5.1.7",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"slugify": "^1.6.6", "slugify": "^1.6.8",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"ua-parser-js": "^2.0.6", "ua-parser-js": "^2.0.9",
"unique-names-generator": "^4.7.1" "unique-names-generator": "^4.7.1"
}, },
"devDependencies": { "devDependencies": {
@@ -31,9 +31,9 @@
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@types/luxon": "^3.7.1", "@types/luxon": "^3.7.1",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/ua-parser-js": "^0.7.39", "@types/ua-parser-js": "^0.7.39",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -8,7 +8,7 @@
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -0,0 +1,72 @@
import { createTable, runClickhouseMigrationCommands } from '../src/clickhouse/migration';
import { getIsCluster, getIsSelfHosting, printBoxMessage } from './helpers';
export async function up() {
const replicatedVersion = '1';
const isClustered = getIsCluster();
const sqls: string[] = [];
sqls.push(
...createTable({
name: 'logs',
columns: [
'`id` UUID DEFAULT generateUUIDv4()',
'`project_id` String CODEC(ZSTD(3))',
'`device_id` String CODEC(ZSTD(3))',
'`profile_id` String CODEC(ZSTD(3))',
'`session_id` String CODEC(LZ4)',
// OpenTelemetry log fields
'`timestamp` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
'`observed_at` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
'`severity_number` UInt8',
'`severity_text` LowCardinality(String)',
'`body` String CODEC(ZSTD(3))',
'`trace_id` String CODEC(ZSTD(3))',
'`span_id` String CODEC(ZSTD(3))',
'`trace_flags` UInt32 DEFAULT 0',
'`logger_name` LowCardinality(String)',
// OTel attributes (log-level key-value pairs)
'`attributes` Map(String, String) CODEC(ZSTD(3))',
// OTel resource attributes (device/app metadata)
'`resource` Map(String, String) CODEC(ZSTD(3))',
// Server-enriched context
'`sdk_name` LowCardinality(String)',
'`sdk_version` LowCardinality(String)',
'`country` LowCardinality(FixedString(2))',
'`city` String',
'`region` LowCardinality(String)',
'`os` LowCardinality(String)',
'`os_version` LowCardinality(String)',
'`browser` LowCardinality(String)',
'`browser_version` LowCardinality(String)',
'`device` LowCardinality(String)',
'`brand` LowCardinality(String)',
'`model` LowCardinality(String)',
],
indices: [
'INDEX idx_severity_number severity_number TYPE minmax GRANULARITY 1',
'INDEX idx_body body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1',
'INDEX idx_trace_id trace_id TYPE bloom_filter GRANULARITY 1',
'INDEX idx_logger_name logger_name TYPE bloom_filter GRANULARITY 1',
],
orderBy: ['project_id', 'toDate(timestamp)', 'severity_number', 'device_id'],
partitionBy: 'toYYYYMM(timestamp)',
settings: {
index_granularity: 8192,
ttl_only_drop_parts: 1,
},
distributionHash: 'cityHash64(project_id, toString(toStartOfHour(timestamp)))',
replicatedVersion,
isClustered,
}),
);
printBoxMessage('Running migration: 13-add-logs', [
'Creates the logs table for OpenTelemetry-compatible device/app log capture.',
]);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}

View File

@@ -13,7 +13,7 @@
"with-env": "dotenv -e ../../.env -c --" "with-env": "dotenv -e ../../.env -c --"
}, },
"dependencies": { "dependencies": {
"@clickhouse/client": "^1.12.1", "@clickhouse/client": "^1.18.2",
"@openpanel/common": "workspace:*", "@openpanel/common": "workspace:*",
"@openpanel/constants": "workspace:*", "@openpanel/constants": "workspace:*",
"@openpanel/json": "workspace:*", "@openpanel/json": "workspace:*",
@@ -21,13 +21,13 @@
"@openpanel/queue": "workspace:^", "@openpanel/queue": "workspace:^",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@prisma/client": "^6.14.0", "@prisma/client": "^6.19.2",
"@prisma/extension-read-replicas": "^0.4.1", "@prisma/extension-read-replicas": "^0.5.0",
"fast-deep-equal": "^3.1.3", "fast-deep-equal": "^3.1.3",
"jiti": "^2.4.1", "jiti": "^2.6.1",
"mathjs": "^12.3.2", "mathjs": "^15.1.1",
"prisma-json-types-generator": "^3.1.1", "prisma-json-types-generator": "^3.6.2",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"uuid": "^9.0.1", "uuid": "^9.0.1",
@@ -36,10 +36,10 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"@types/uuid": "^9.0.8", "@types/uuid": "^11.0.0",
"prisma": "^6.14.0", "prisma": "^6.19.2",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -1,6 +1,7 @@
import { getRedisCache } from '@openpanel/redis'; import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { ch } from '../clickhouse/client'; import * as chClient from '../clickhouse/client';
const { ch } = chClient;
// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer // Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
vi.mock('../services/event.service', () => ({})); vi.mock('../services/event.service', () => ({}));
@@ -10,7 +11,8 @@ import { EventBuffer } from './event-buffer';
const redis = getRedisCache(); const redis = getRedisCache();
beforeEach(async () => { beforeEach(async () => {
await redis.flushdb(); const keys = await redis.keys('event*');
if (keys.length > 0) await redis.del(...keys);
}); });
afterAll(async () => { afterAll(async () => {
@@ -209,18 +211,16 @@ describe('EventBuffer', () => {
}); });
it('tracks active visitors', async () => { it('tracks active visitors', async () => {
const event = { const querySpy = vi
project_id: 'p9', .spyOn(chClient, 'chQuery')
profile_id: 'u9', .mockResolvedValueOnce([{ count: 2 }] as any);
name: 'custom',
created_at: new Date().toISOString(),
} as any;
eventBuffer.add(event);
await eventBuffer.flush();
const count = await eventBuffer.getActiveVisitorCount('p9'); const count = await eventBuffer.getActiveVisitorCount('p9');
expect(count).toBeGreaterThanOrEqual(1); expect(count).toBe(2);
expect(querySpy).toHaveBeenCalledOnce();
expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");
querySpy.mockRestore();
}); });
it('handles multiple sessions independently — all events go to buffer', async () => { it('handles multiple sessions independently — all events go to buffer', async () => {
@@ -273,4 +273,24 @@ describe('EventBuffer', () => {
expect(await eventBuffer.getBufferSize()).toBe(5); expect(await eventBuffer.getBufferSize()).toBe(5);
}); });
it('retains events in queue when ClickHouse insert fails', async () => {
eventBuffer.add({
project_id: 'p12',
name: 'event1',
created_at: new Date().toISOString(),
} as any);
await eventBuffer.flush();
const insertSpy = vi
.spyOn(ch, 'insert')
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await eventBuffer.processBuffer();
// Events must still be in the queue — not lost
expect(await eventBuffer.getBufferSize()).toBe(1);
insertSpy.mockRestore();
});
}); });

View File

@@ -1,6 +1,6 @@
import { getSafeJson } from '@openpanel/json'; import { getSafeJson } from '@openpanel/json';
import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis'; import { getRedisCache, publishEvent } from '@openpanel/redis';
import { ch } from '../clickhouse/client'; import { ch, chQuery } from '../clickhouse/client';
import type { IClickhouseEvent } from '../services/event.service'; import type { IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer'; import { BaseBuffer } from './base-buffer';
@@ -25,10 +25,6 @@ export class EventBuffer extends BaseBuffer {
/** Tracks consecutive flush failures for observability; reset on success. */ /** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0; private flushRetryCount = 0;
private activeVisitorsExpiration = 60 * 5; // 5 minutes
/** How often (ms) we refresh the heartbeat key + zadd per visitor. */
private heartbeatRefreshMs = 60_000; // 1 minute
private lastHeartbeat = new Map<string, number>();
private queueKey = 'event_buffer:queue'; private queueKey = 'event_buffer:queue';
protected bufferCounterKey = 'event_buffer:total_count'; protected bufferCounterKey = 'event_buffer:total_count';
@@ -87,20 +83,12 @@ export class EventBuffer extends BaseBuffer {
for (const event of eventsToFlush) { for (const event of eventsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(event)); multi.rpush(this.queueKey, JSON.stringify(event));
if (event.profile_id) {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id
);
}
} }
multi.incrby(this.bufferCounterKey, eventsToFlush.length); multi.incrby(this.bufferCounterKey, eventsToFlush.length);
await multi.exec(); await multi.exec();
this.flushRetryCount = 0; this.flushRetryCount = 0;
this.pruneHeartbeatMap();
} catch (error) { } catch (error) {
// Re-queue failed events at the front to preserve order and avoid data loss // Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents); this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -202,58 +190,21 @@ export class EventBuffer extends BaseBuffer {
} }
} }
public async getBufferSize() { public getBufferSize() {
return this.getBufferSizeWithCounter(async () => { return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache(); const redis = getRedisCache();
return await redis.llen(this.queueKey); return await redis.llen(this.queueKey);
}); });
} }
private pruneHeartbeatMap() {
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
for (const [key, ts] of this.lastHeartbeat) {
if (ts < cutoff) {
this.lastHeartbeat.delete(key);
}
}
}
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string
) {
const key = `${projectId}:${profileId}`;
const now = Date.now();
const last = this.lastHeartbeat.get(key) ?? 0;
if (now - last < this.heartbeatRefreshMs) {
return;
}
this.lastHeartbeat.set(key, now);
const zsetKey = `live:visitors:${projectId}`;
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
multi
.zadd(zsetKey, now, profileId)
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
}
public async getActiveVisitorCount(projectId: string): Promise<number> { public async getActiveVisitorCount(projectId: string): Promise<number> {
const redis = getRedisCache(); const rows = await chQuery<{ count: number }>(
const zsetKey = `live:visitors:${projectId}`; `SELECT uniq(profile_id) AS count
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000; FROM events
WHERE project_id = '${projectId}'
const multi = redis.multi(); AND profile_id != ''
multi AND created_at >= now() - INTERVAL 5 MINUTE`
.zremrangebyscore(zsetKey, '-inf', cutoff) );
.zcount(zsetKey, cutoff, '+inf'); return rows[0]?.count ?? 0;
const [, count] = (await multi.exec()) as [
[Error | null, any],
[Error | null, number],
];
return count[1] || 0;
} }
} }

View File

@@ -1,6 +1,7 @@
import { BotBuffer as BotBufferRedis } from './bot-buffer'; import { BotBuffer as BotBufferRedis } from './bot-buffer';
import { EventBuffer as EventBufferRedis } from './event-buffer'; import { EventBuffer as EventBufferRedis } from './event-buffer';
import { GroupBuffer } from './group-buffer'; import { GroupBuffer } from './group-buffer';
import { LogBuffer } from './log-buffer';
import { ProfileBackfillBuffer } from './profile-backfill-buffer'; import { ProfileBackfillBuffer } from './profile-backfill-buffer';
import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer'; import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
import { ReplayBuffer } from './replay-buffer'; import { ReplayBuffer } from './replay-buffer';
@@ -13,6 +14,8 @@ export const sessionBuffer = new SessionBuffer();
export const profileBackfillBuffer = new ProfileBackfillBuffer(); export const profileBackfillBuffer = new ProfileBackfillBuffer();
export const replayBuffer = new ReplayBuffer(); export const replayBuffer = new ReplayBuffer();
export const groupBuffer = new GroupBuffer(); export const groupBuffer = new GroupBuffer();
export const logBuffer = new LogBuffer();
export type { ProfileBackfillEntry } from './profile-backfill-buffer'; export type { ProfileBackfillEntry } from './profile-backfill-buffer';
export type { IClickhouseSessionReplayChunk } from './replay-buffer'; export type { IClickhouseSessionReplayChunk } from './replay-buffer';
export type { IClickhouseLog } from './log-buffer';

View File

@@ -0,0 +1,193 @@
import { getSafeJson } from '@openpanel/json';
import { getRedisCache } from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import { BaseBuffer } from './base-buffer';
export interface IClickhouseLog {
id?: string;
project_id: string;
device_id: string;
profile_id: string;
session_id: string;
timestamp: string;
observed_at: string;
severity_number: number;
severity_text: string;
body: string;
trace_id: string;
span_id: string;
trace_flags: number;
logger_name: string;
attributes: Record<string, string>;
resource: Record<string, string>;
sdk_name: string;
sdk_version: string;
country: string;
city: string;
region: string;
os: string;
os_version: string;
browser: string;
browser_version: string;
device: string;
brand: string;
model: string;
}
export class LogBuffer extends BaseBuffer {
private batchSize = process.env.LOG_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.LOG_BUFFER_BATCH_SIZE, 10)
: 4000;
private chunkSize = process.env.LOG_BUFFER_CHUNK_SIZE
? Number.parseInt(process.env.LOG_BUFFER_CHUNK_SIZE, 10)
: 1000;
private microBatchIntervalMs = process.env.LOG_BUFFER_MICRO_BATCH_MS
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_MS, 10)
: 10;
private microBatchMaxSize = process.env.LOG_BUFFER_MICRO_BATCH_SIZE
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_SIZE, 10)
: 100;
private pendingLogs: IClickhouseLog[] = [];
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private isFlushing = false;
private flushRetryCount = 0;
private queueKey = 'log_buffer:queue';
protected bufferCounterKey = 'log_buffer:total_count';
constructor() {
super({
name: 'log',
onFlush: async () => {
await this.processBuffer();
},
});
}
add(log: IClickhouseLog) {
this.pendingLogs.push(log);
if (this.pendingLogs.length >= this.microBatchMaxSize) {
this.flushLocalBuffer();
return;
}
if (!this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
public async flush() {
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
await this.flushLocalBuffer();
}
private async flushLocalBuffer() {
if (this.isFlushing || this.pendingLogs.length === 0) {
return;
}
this.isFlushing = true;
const logsToFlush = this.pendingLogs;
this.pendingLogs = [];
try {
const redis = getRedisCache();
const multi = redis.multi();
for (const log of logsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(log));
}
multi.incrby(this.bufferCounterKey, logsToFlush.length);
await multi.exec();
this.flushRetryCount = 0;
} catch (error) {
this.pendingLogs = logsToFlush.concat(this.pendingLogs);
this.flushRetryCount += 1;
this.logger.warn('Failed to flush log buffer to Redis; logs re-queued', {
error,
logCount: logsToFlush.length,
flushRetryCount: this.flushRetryCount,
});
} finally {
this.isFlushing = false;
if (this.pendingLogs.length > 0 && !this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
}
async processBuffer() {
const redis = getRedisCache();
try {
const queueLogs = await redis.lrange(this.queueKey, 0, this.batchSize - 1);
if (queueLogs.length === 0) {
this.logger.debug('No logs to process');
return;
}
const logsToClickhouse: IClickhouseLog[] = [];
for (const logStr of queueLogs) {
const log = getSafeJson<IClickhouseLog>(logStr);
if (log) {
logsToClickhouse.push(log);
}
}
if (logsToClickhouse.length === 0) {
this.logger.debug('No valid logs to process');
return;
}
logsToClickhouse.sort(
(a, b) =>
new Date(a.timestamp || 0).getTime() -
new Date(b.timestamp || 0).getTime(),
);
this.logger.info('Inserting logs into ClickHouse', {
totalLogs: logsToClickhouse.length,
chunks: Math.ceil(logsToClickhouse.length / this.chunkSize),
});
for (const chunk of this.chunks(logsToClickhouse, this.chunkSize)) {
await ch.insert({
table: 'logs',
values: chunk,
format: 'JSONEachRow',
});
}
await redis
.multi()
.ltrim(this.queueKey, queueLogs.length, -1)
.decrby(this.bufferCounterKey, queueLogs.length)
.exec();
this.logger.info('Processed logs from Redis buffer', {
batchSize: this.batchSize,
logsProcessed: logsToClickhouse.length,
});
} catch (error) {
this.logger.error('Error processing log Redis buffer', { error });
}
}
public getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache();
return await redis.llen(this.queueKey);
});
}
}

View File

@@ -1,6 +1,5 @@
import { getRedisCache } from '@openpanel/redis'; import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { getSafeJson } from '@openpanel/json';
import type { IClickhouseProfile } from '../services/profile.service'; import type { IClickhouseProfile } from '../services/profile.service';
// Mock chQuery to avoid hitting real ClickHouse // Mock chQuery to avoid hitting real ClickHouse
@@ -36,7 +35,11 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile
} }
beforeEach(async () => { beforeEach(async () => {
await redis.flushdb(); const keys = [
...await redis.keys('profile*'),
...await redis.keys('lock:profile'),
];
if (keys.length > 0) await redis.del(...keys);
vi.mocked(chQuery).mockResolvedValue([]); vi.mocked(chQuery).mockResolvedValue([]);
}); });
@@ -63,64 +66,12 @@ describe('ProfileBuffer', () => {
expect(sizeAfter).toBe(sizeBefore + 1); expect(sizeAfter).toBe(sizeBefore + 1);
}); });
it('merges subsequent updates via cache (sequential calls)', async () => { it('concurrent adds: both raw profiles are queued', async () => {
const identifyProfile = makeProfile({ const identifyProfile = makeProfile({
first_name: 'John', first_name: 'John',
email: 'john@example.com', email: 'john@example.com',
groups: [], groups: [],
}); });
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Sequential: identify first, then group
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
// Second add should read the cached identify profile and merge groups in
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent identify + group calls preserve all data', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
// Both calls run concurrently — the per-profile lock serializes them so the
// second one reads the first's result from cache and merges correctly.
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('race condition: concurrent writes produce one merged buffer entry', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({ const groupProfile = makeProfile({
first_name: '', first_name: '',
email: '', email: '',
@@ -128,24 +79,126 @@ describe('ProfileBuffer', () => {
}); });
const sizeBefore = await profileBuffer.getBufferSize(); const sizeBefore = await profileBuffer.getBufferSize();
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
const sizeAfter = await profileBuffer.getBufferSize();
// Both raw profiles are queued; merge happens at flush time
expect(sizeAfter).toBe(sizeBefore + 2);
});
it('merges sequential updates for the same profile at flush time', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
it('merges concurrent updates for the same profile at flush time', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
email: 'john@example.com',
groups: [],
});
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
await Promise.all([ await Promise.all([
profileBuffer.add(identifyProfile), profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile), profileBuffer.add(groupProfile),
]); ]);
await profileBuffer.processBuffer();
const sizeAfter = await profileBuffer.getBufferSize(); const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('john@example.com');
expect(cached?.groups).toContain('group-abc');
});
// The second add merges into the first — only 2 buffer entries total it('uses existing ClickHouse data for cache misses when merging', async () => {
// (one from identify, one merged update with group) const existingInClickhouse = makeProfile({
expect(sizeAfter).toBe(sizeBefore + 2); first_name: 'Jane',
email: 'jane@example.com',
groups: ['existing-group'],
});
vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);
// The last entry in the buffer should have both name and group const incomingProfile = makeProfile({
const rawEntries = await redis.lrange('profile-buffer', 0, -1); first_name: '',
const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e)); email: '',
const lastEntry = entries[entries.length - 1]; groups: ['new-group'],
});
expect(lastEntry?.first_name).toBe('John'); await profileBuffer.add(incomingProfile);
expect(lastEntry?.groups).toContain('group-abc'); await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('Jane');
expect(cached?.email).toBe('jane@example.com');
expect(cached?.groups).toContain('existing-group');
expect(cached?.groups).toContain('new-group');
});
it('buffer is empty after flush', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
expect(await profileBuffer.getBufferSize()).toBe(1);
await profileBuffer.processBuffer();
expect(await profileBuffer.getBufferSize()).toBe(0);
});
it('retains profiles in queue when ClickHouse insert fails', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
const { ch } = await import('../clickhouse/client');
const insertSpy = vi
.spyOn(ch, 'insert')
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await profileBuffer.processBuffer();
// Profiles must still be in the queue — not lost
expect(await profileBuffer.getBufferSize()).toBe(1);
insertSpy.mockRestore();
});
it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {
vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));
const { ch } = await import('../clickhouse/client');
const insertSpy = vi
.spyOn(ch, 'insert')
.mockResolvedValueOnce(undefined as any);
await profileBuffer.add(makeProfile({ first_name: 'John' }));
await profileBuffer.processBuffer();
// Insert must still have been called — no data loss even when fetch fails
expect(insertSpy).toHaveBeenCalled();
expect(await profileBuffer.getBufferSize()).toBe(0);
insertSpy.mockRestore();
}); });
}); });

View File

@@ -1,9 +1,6 @@
import { deepMergeObjects } from '@openpanel/common'; import { deepMergeObjects } from '@openpanel/common';
import { generateSecureId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json'; import { getSafeJson } from '@openpanel/json';
import type { ILogger } from '@openpanel/logger';
import { getRedisCache, type Redis } from '@openpanel/redis'; import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import { omit, uniq } from 'ramda'; import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring'; import sqlstring from 'sqlstring';
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client'; import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
@@ -11,29 +8,24 @@ import type { IClickhouseProfile } from '../services/profile.service';
import { BaseBuffer } from './base-buffer'; import { BaseBuffer } from './base-buffer';
export class ProfileBuffer extends BaseBuffer { export class ProfileBuffer extends BaseBuffer {
private batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10) ? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)
: 200; : 200;
private chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE private readonly chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10) ? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10)
: 1000; : 1000;
private ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS private readonly ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
? Number.parseInt(process.env.PROFILE_BUFFER_TTL_IN_SECONDS, 10) ? Number.parseInt(process.env.PROFILE_BUFFER_TTL_IN_SECONDS, 10)
: 60 * 60; : 60 * 60;
/** Max profiles per ClickHouse IN-clause fetch to keep query size bounded */
private readonly fetchChunkSize = process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE, 10)
: 50;
private readonly redisKey = 'profile-buffer'; private readonly redisKey = 'profile-buffer';
private readonly redisProfilePrefix = 'profile-cache:'; private readonly redisProfilePrefix = 'profile-cache:';
private redis: Redis; private readonly redis: Redis;
private releaseLockSha: string | null = null;
private readonly releaseLockScript = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
else
return 0
end
`;
constructor() { constructor() {
super({ super({
@@ -43,9 +35,6 @@ export class ProfileBuffer extends BaseBuffer {
}, },
}); });
this.redis = getRedisCache(); this.redis = getRedisCache();
this.redis.script('LOAD', this.releaseLockScript).then((sha) => {
this.releaseLockSha = sha as string;
});
} }
private getProfileCacheKey({ private getProfileCacheKey({
@@ -58,243 +47,236 @@ export class ProfileBuffer extends BaseBuffer {
return `${this.redisProfilePrefix}${projectId}:${profileId}`; return `${this.redisProfilePrefix}${projectId}:${profileId}`;
} }
private async withProfileLock<T>( public async fetchFromCache(
profileId: string, profileId: string,
projectId: string, projectId: string
fn: () => Promise<T> ): Promise<IClickhouseProfile | null> {
): Promise<T> { const cacheKey = this.getProfileCacheKey({ profileId, projectId });
const lockKey = `profile-lock:${projectId}:${profileId}`; const cached = await this.redis.get(cacheKey);
const lockId = generateSecureId('lock'); if (!cached) {
const maxRetries = 20; return null;
const retryDelayMs = 50;
for (let i = 0; i < maxRetries; i++) {
const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');
if (acquired === 'OK') {
try {
return await fn();
} finally {
if (this.releaseLockSha) {
await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);
} else {
await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);
}
}
}
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
} }
return getSafeJson<IClickhouseProfile>(cached);
this.logger.error(
'Failed to acquire profile lock, proceeding without lock',
{
profileId,
projectId,
}
);
return fn();
}
async alreadyExists(profile: IClickhouseProfile) {
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
return (await this.redis.exists(cacheKey)) === 1;
} }
async add(profile: IClickhouseProfile, isFromEvent = false) { async add(profile: IClickhouseProfile, isFromEvent = false) {
const logger = this.logger.child({
projectId: profile.project_id,
profileId: profile.id,
});
try { try {
logger.debug('Adding profile'); if (isFromEvent) {
if (isFromEvent && (await this.alreadyExists(profile))) {
logger.debug('Profile already created, skipping');
return;
}
await this.withProfileLock(profile.id, profile.project_id, async () => {
const existingProfile = await this.fetchProfile(profile, logger);
// Delete any properties that are not server related if we have a non-server profile
if (
existingProfile?.properties.device !== 'server' &&
profile.properties.device === 'server'
) {
profile.properties = omit(
[
'city',
'country',
'region',
'longitude',
'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
profile.properties
);
}
const mergedProfile: IClickhouseProfile = existingProfile
? {
...deepMergeObjects(
existingProfile,
omit(['created_at', 'groups'], profile)
),
groups: uniq([
...(existingProfile.groups ?? []),
...(profile.groups ?? []),
]),
}
: profile;
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
this.logger.debug('Merged profile will be inserted', {
mergedProfile,
existingProfile,
profile,
});
const cacheKey = this.getProfileCacheKey({ const cacheKey = this.getProfileCacheKey({
profileId: profile.id, profileId: profile.id,
projectId: profile.project_id, projectId: profile.project_id,
}); });
const exists = await this.redis.exists(cacheKey);
const result = await this.redis if (exists === 1) {
.multi()
.set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(mergedProfile))
.incr(this.bufferCounterKey)
.llen(this.redisKey)
.exec();
if (!result) {
this.logger.error('Failed to add profile to Redis', {
profile,
cacheKey,
});
return; return;
} }
const bufferLength = (result?.[3]?.[1] as number) ?? 0; }
this.logger.debug('Current buffer length', { const result = await this.redis
bufferLength, .multi()
batchSize: this.batchSize, .rpush(this.redisKey, JSON.stringify(profile))
}); .incr(this.bufferCounterKey)
if (bufferLength >= this.batchSize) { .llen(this.redisKey)
await this.tryFlush(); .exec();
}
}); if (!result) {
this.logger.error('Failed to add profile to Redis', { profile });
return;
}
const bufferLength = (result?.[2]?.[1] as number) ?? 0;
if (bufferLength >= this.batchSize) {
await this.tryFlush();
}
} catch (error) { } catch (error) {
this.logger.error('Failed to add profile', { error, profile }); this.logger.error('Failed to add profile', { error, profile });
} }
} }
private async fetchProfile( private mergeProfiles(
profile: IClickhouseProfile, existing: IClickhouseProfile | null,
logger: ILogger incoming: IClickhouseProfile
): Promise<IClickhouseProfile | null> { ): IClickhouseProfile {
const existingProfile = await this.fetchFromCache( if (!existing) {
profile.id, return incoming;
profile.project_id
);
if (existingProfile) {
logger.debug('Profile found in Redis');
return existingProfile;
} }
return this.fetchFromClickhouse(profile, logger); let profile = incoming;
} if (
existing.properties.device !== 'server' &&
public async fetchFromCache( incoming.properties.device === 'server'
profileId: string, ) {
projectId: string profile = {
): Promise<IClickhouseProfile | null> { ...incoming,
const cacheKey = this.getProfileCacheKey({ properties: omit(
profileId, [
projectId, 'city',
}); 'country',
const existingProfile = await this.redis.get(cacheKey); 'region',
if (!existingProfile) { 'longitude',
return null; 'latitude',
'os',
'osVersion',
'browser',
'device',
'isServer',
'os_version',
'browser_version',
],
incoming.properties
),
};
} }
return getSafeJson<IClickhouseProfile>(existingProfile);
return {
...deepMergeObjects(existing, omit(['created_at', 'groups'], profile)),
groups: uniq([...(existing.groups ?? []), ...(incoming.groups ?? [])]),
};
} }
private async fetchFromClickhouse( private async batchFetchFromClickhouse(
profile: IClickhouseProfile, profiles: IClickhouseProfile[]
logger: ILogger ): Promise<Map<string, IClickhouseProfile>> {
): Promise<IClickhouseProfile | null> { const result = new Map<string, IClickhouseProfile>();
logger.debug('Fetching profile from Clickhouse');
const result = await chQuery<IClickhouseProfile>( // Non-external (anonymous/device) profiles get a 2-day recency filter to
`SELECT // avoid pulling stale anonymous sessions from far back.
id, const external = profiles.filter((p) => p.is_external !== false);
project_id, const nonExternal = profiles.filter((p) => p.is_external === false);
last_value(nullIf(first_name, '')) as first_name,
last_value(nullIf(last_name, '')) as last_name, const fetchGroup = async (
last_value(nullIf(email, '')) as email, group: IClickhouseProfile[],
last_value(nullIf(avatar, '')) as avatar, withDateFilter: boolean
last_value(is_external) as is_external, ) => {
last_value(properties) as properties, for (const chunk of this.chunks(group, this.fetchChunkSize)) {
last_value(created_at) as created_at const tuples = chunk
FROM ${TABLE_NAMES.profiles} .map(
WHERE (p) =>
id = ${sqlstring.escape(String(profile.id))} AND `(${sqlstring.escape(String(p.id))}, ${sqlstring.escape(p.project_id)})`
project_id = ${sqlstring.escape(profile.project_id)} )
${ .join(', ');
profile.is_external === false try {
? ' AND profiles.created_at > now() - INTERVAL 2 DAY' const rows = await chQuery<IClickhouseProfile>(
: '' `SELECT
id,
project_id,
argMax(nullIf(first_name, ''), ${TABLE_NAMES.profiles}.created_at) as first_name,
argMax(nullIf(last_name, ''), ${TABLE_NAMES.profiles}.created_at) as last_name,
argMax(nullIf(email, ''), ${TABLE_NAMES.profiles}.created_at) as email,
argMax(nullIf(avatar, ''), ${TABLE_NAMES.profiles}.created_at) as avatar,
argMax(is_external, ${TABLE_NAMES.profiles}.created_at) as is_external,
argMax(properties, ${TABLE_NAMES.profiles}.created_at) as properties,
max(created_at) as created_at
FROM ${TABLE_NAMES.profiles}
WHERE (id, project_id) IN (${tuples})
${withDateFilter ? `AND ${TABLE_NAMES.profiles}.created_at > now() - INTERVAL 2 DAY` : ''}
GROUP BY id, project_id`
);
for (const row of rows) {
result.set(`${row.project_id}:${row.id}`, row);
}
} catch (error) {
this.logger.warn(
'Failed to batch fetch profiles from Clickhouse, proceeding without existing data',
{ error, chunkSize: chunk.length }
);
} }
GROUP BY id, project_id }
ORDER BY created_at DESC };
LIMIT 1`
); await Promise.all([
logger.debug('Clickhouse fetch result', { fetchGroup(external, false),
found: !!result[0], fetchGroup(nonExternal, true),
}); ]);
return result[0] || null;
return result;
} }
async processBuffer() { async processBuffer() {
try { try {
this.logger.debug('Starting profile buffer processing'); this.logger.debug('Starting profile buffer processing');
const profiles = await this.redis.lrange( const rawProfiles = await this.redis.lrange(
this.redisKey, this.redisKey,
0, 0,
this.batchSize - 1 this.batchSize - 1
); );
if (profiles.length === 0) { if (rawProfiles.length === 0) {
this.logger.debug('No profiles to process'); this.logger.debug('No profiles to process');
return; return;
} }
this.logger.debug(`Processing ${profiles.length} profiles in buffer`); const parsedProfiles = rawProfiles
const parsedProfiles = profiles.map((p) => .map((p) => getSafeJson<IClickhouseProfile>(p))
getSafeJson<IClickhouseProfile>(p) .filter(Boolean) as IClickhouseProfile[];
);
for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) { // Merge within batch: collapse multiple updates for the same profile
const mergedInBatch = new Map<string, IClickhouseProfile>();
for (const profile of parsedProfiles) {
const key = `${profile.project_id}:${profile.id}`;
mergedInBatch.set(
key,
this.mergeProfiles(mergedInBatch.get(key) ?? null, profile)
);
}
const uniqueProfiles = Array.from(mergedInBatch.values());
// Check Redis cache for all unique profiles in a single MGET
const cacheKeys = uniqueProfiles.map((p) =>
this.getProfileCacheKey({ profileId: p.id, projectId: p.project_id })
);
const cacheResults = await this.redis.mget(...cacheKeys);
const existingByKey = new Map<string, IClickhouseProfile>();
const cacheMisses: IClickhouseProfile[] = [];
for (let i = 0; i < uniqueProfiles.length; i++) {
const uniqueProfile = uniqueProfiles[i];
if (uniqueProfile) {
const key = `${uniqueProfile.project_id}:${uniqueProfile.id}`;
const cached = cacheResults[i]
? getSafeJson<IClickhouseProfile>(cacheResults[i]!)
: null;
if (cached) {
existingByKey.set(key, cached);
} else {
cacheMisses.push(uniqueProfile);
}
}
}
// Fetch cache misses from ClickHouse in bounded chunks
if (cacheMisses.length > 0) {
const clickhouseResults =
await this.batchFetchFromClickhouse(cacheMisses);
for (const [key, profile] of clickhouseResults) {
existingByKey.set(key, profile);
}
}
// Final merge: in-batch profile + existing (from cache or ClickHouse)
const toInsert: IClickhouseProfile[] = [];
const multi = this.redis.multi();
for (const profile of uniqueProfiles) {
const key = `${profile.project_id}:${profile.id}`;
const merged = this.mergeProfiles(
existingByKey.get(key) ?? null,
profile
);
toInsert.push(merged);
multi.set(
this.getProfileCacheKey({
projectId: profile.project_id,
profileId: profile.id,
}),
JSON.stringify(merged),
'EX',
this.ttlInSeconds
);
}
for (const chunk of this.chunks(toInsert, this.chunkSize)) {
await ch.insert({ await ch.insert({
table: TABLE_NAMES.profiles, table: TABLE_NAMES.profiles,
values: chunk, values: chunk,
@@ -302,22 +284,21 @@ export class ProfileBuffer extends BaseBuffer {
}); });
} }
// Only remove profiles after successful insert and update counter multi
await this.redis .ltrim(this.redisKey, rawProfiles.length, -1)
.multi() .decrby(this.bufferCounterKey, rawProfiles.length);
.ltrim(this.redisKey, profiles.length, -1) await multi.exec();
.decrby(this.bufferCounterKey, profiles.length)
.exec();
this.logger.debug('Successfully completed profile processing', { this.logger.debug('Successfully completed profile processing', {
totalProfiles: profiles.length, totalProfiles: rawProfiles.length,
uniqueProfiles: uniqueProfiles.length,
}); });
} catch (error) { } catch (error) {
this.logger.error('Failed to process buffer', { error }); this.logger.error('Failed to process buffer', { error });
} }
} }
async getBufferSize() { getBufferSize() {
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey)); return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
} }
} }

View File

@@ -0,0 +1,122 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { ch } from '../clickhouse/client';
vi.mock('../clickhouse/client', () => ({
ch: {
insert: vi.fn().mockResolvedValue(undefined),
},
TABLE_NAMES: {
sessions: 'sessions',
},
}));
import { SessionBuffer } from './session-buffer';
import type { IClickhouseEvent } from '../services/event.service';
const redis = getRedisCache();
function makeEvent(overrides: Partial<IClickhouseEvent>): IClickhouseEvent {
return {
id: 'event-1',
project_id: 'project-1',
profile_id: 'profile-1',
device_id: 'device-1',
session_id: 'session-1',
name: 'screen_view',
path: '/home',
origin: '',
referrer: '',
referrer_name: '',
referrer_type: '',
duration: 0,
properties: {},
created_at: new Date().toISOString(),
groups: [],
...overrides,
} as IClickhouseEvent;
}
beforeEach(async () => {
const keys = [
...await redis.keys('session*'),
...await redis.keys('lock:session'),
];
if (keys.length > 0) await redis.del(...keys);
vi.mocked(ch.insert).mockResolvedValue(undefined as any);
});
afterAll(async () => {
try {
await redis.quit();
} catch {}
});
describe('SessionBuffer', () => {
let sessionBuffer: SessionBuffer;
beforeEach(() => {
sessionBuffer = new SessionBuffer();
});
it('adds a new session to the buffer', async () => {
const sizeBefore = await sessionBuffer.getBufferSize();
await sessionBuffer.add(makeEvent({}));
const sizeAfter = await sessionBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore + 1);
});
it('skips session_start and session_end events', async () => {
const sizeBefore = await sessionBuffer.getBufferSize();
await sessionBuffer.add(makeEvent({ name: 'session_start' }));
await sessionBuffer.add(makeEvent({ name: 'session_end' }));
const sizeAfter = await sessionBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore);
});
it('updates existing session on subsequent events', async () => {
const t0 = Date.now();
await sessionBuffer.add(makeEvent({ created_at: new Date(t0).toISOString() }));
// Second event updates the same session — emits old (sign=-1) + new (sign=1)
const sizeBefore = await sessionBuffer.getBufferSize();
await sessionBuffer.add(makeEvent({ created_at: new Date(t0 + 5000).toISOString() }));
const sizeAfter = await sessionBuffer.getBufferSize();
expect(sizeAfter).toBe(sizeBefore + 2);
});
it('processes buffer and inserts sessions into ClickHouse', async () => {
await sessionBuffer.add(makeEvent({}));
const insertSpy = vi
.spyOn(ch, 'insert')
.mockResolvedValueOnce(undefined as any);
await sessionBuffer.processBuffer();
expect(insertSpy).toHaveBeenCalledWith(
expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
);
expect(await sessionBuffer.getBufferSize()).toBe(0);
insertSpy.mockRestore();
});
it('retains sessions in queue when ClickHouse insert fails', async () => {
await sessionBuffer.add(makeEvent({}));
const insertSpy = vi
.spyOn(ch, 'insert')
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await sessionBuffer.processBuffer();
// Sessions must still be in the queue — not lost
expect(await sessionBuffer.getBufferSize()).toBe(1);
insertSpy.mockRestore();
});
});

View File

@@ -9,10 +9,10 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@react-email/components": "^0.5.6", "@react-email/components": "^1.0.10",
"react": "catalog:", "react": "catalog:",
"react-dom": "catalog:", "react-dom": "catalog:",
"resend": "^4.0.1", "resend": "^4.8.0",
"responsive-react-email": "^0.0.5", "responsive-react-email": "^0.0.5",
"zod": "catalog:" "zod": "catalog:"
}, },

View File

@@ -7,15 +7,15 @@
"codegen": "jiti scripts/download.ts" "codegen": "jiti scripts/download.ts"
}, },
"dependencies": { "dependencies": {
"@maxmind/geoip2-node": "^6.1.0", "@maxmind/geoip2-node": "^6.3.4",
"lru-cache": "^11.2.2" "lru-cache": "^11.2.7"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"fast-extract": "^1.4.3", "fast-extract": "^1.14.2",
"jiti": "^2.4.1", "jiti": "^2.6.1",
"tar": "^7.4.3", "tar": "^7.5.13",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -18,18 +18,18 @@
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/queue": "workspace:*", "@openpanel/queue": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"csv-parse": "^6.1.0", "csv-parse": "^6.2.1",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"uuid": "^9.0.1", "uuid": "^9.0.1",
"zod": "catalog:" "zod": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/logger": "workspace:*", "@openpanel/logger": "workspace:*",
"@types/node": "^20.0.0", "@types/node": "^20.19.37",
"@types/ramda": "^0.31.1", "@types/ramda": "^0.31.1",
"@types/uuid": "^9.0.7", "@types/uuid": "^11.0.0",
"bullmq": "^5.8.7", "bullmq": "^5.71.1",
"typescript": "^5.0.0", "typescript": "^5.9.3",
"vitest": "^1.0.0" "vitest": "^1.6.1"
} }
} }

View File

@@ -7,8 +7,8 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@slack/bolt": "^3.18.0", "@slack/bolt": "^3.22.0",
"@slack/oauth": "^3.0.0" "@slack/oauth": "^3.0.5"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",

View File

@@ -11,11 +11,11 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@babel/parser": "^7.26.0" "@babel/parser": "^7.29.2"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"typescript": "catalog:", "typescript": "catalog:",
"vitest": "^2.1.8" "vitest": "^2.1.9"
} }
} }

View File

@@ -7,13 +7,13 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@hyperdx/node-opentelemetry": "^0.8.1", "@hyperdx/node-opentelemetry": "^0.10.3",
"winston": "^3.14.2" "winston": "^3.19.0"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -11,16 +11,16 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@polar-sh/sdk": "^0.35.4" "@polar-sh/sdk": "^0.46.7"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/inquirer": "^9.0.7", "@types/inquirer": "^9.0.9",
"@types/inquirer-autocomplete-prompt": "^3.0.3", "@types/inquirer-autocomplete-prompt": "^3.0.3",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/react": "catalog:", "@types/react": "catalog:",
"inquirer": "^9.3.5", "inquirer": "^9.3.8",
"inquirer-autocomplete-prompt": "^3.0.1", "inquirer-autocomplete-prompt": "^3.0.1",
"typescript": "catalog:" "typescript": "catalog:"
} }

View File

@@ -10,7 +10,7 @@
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/logger": "workspace:*", "@openpanel/logger": "workspace:*",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"bullmq": "^5.63.0", "bullmq": "^5.71.1",
"groupmq": "catalog:" "groupmq": "catalog:"
}, },
"devDependencies": { "devDependencies": {

View File

@@ -8,7 +8,7 @@ import { createLogger } from '@openpanel/logger';
import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis'; import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
import { Queue } from 'bullmq'; import { Queue } from 'bullmq';
import { Queue as GroupQueue } from 'groupmq'; import { Queue as GroupQueue } from 'groupmq';
import type { ITrackPayload } from '../../validation'; import type { ILogPayload, ITrackPayload } from '../../validation';
export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt( export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(
process.env.EVENTS_GROUP_QUEUES_SHARDS || '1', process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',
@@ -138,6 +138,10 @@ export type CronQueuePayloadFlushGroups = {
type: 'flushGroups'; type: 'flushGroups';
payload: undefined; payload: undefined;
}; };
export type CronQueuePayloadFlushLogs = {
type: 'flushLogs';
payload: undefined;
};
export type CronQueuePayload = export type CronQueuePayload =
| CronQueuePayloadSalt | CronQueuePayloadSalt
| CronQueuePayloadFlushEvents | CronQueuePayloadFlushEvents
@@ -146,6 +150,7 @@ export type CronQueuePayload =
| CronQueuePayloadFlushProfileBackfill | CronQueuePayloadFlushProfileBackfill
| CronQueuePayloadFlushReplay | CronQueuePayloadFlushReplay
| CronQueuePayloadFlushGroups | CronQueuePayloadFlushGroups
| CronQueuePayloadFlushLogs
| CronQueuePayloadPing | CronQueuePayloadPing
| CronQueuePayloadProject | CronQueuePayloadProject
| CronQueuePayloadInsightsDaily | CronQueuePayloadInsightsDaily
@@ -297,3 +302,50 @@ export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {
removeOnFail: 100, removeOnFail: 100,
}, },
}); });
export type LogsQueuePayload = {
type: 'incomingLog';
payload: {
projectId: string;
log: ILogPayload & {
timestamp: string;
};
uaInfo:
| {
readonly isServer: true;
readonly device: 'server';
readonly os: '';
readonly osVersion: '';
readonly browser: '';
readonly browserVersion: '';
readonly brand: '';
readonly model: '';
}
| {
readonly os: string | undefined;
readonly osVersion: string | undefined;
readonly browser: string | undefined;
readonly browserVersion: string | undefined;
readonly device: string;
readonly brand: string | undefined;
readonly model: string | undefined;
readonly isServer: false;
};
geo: {
country: string | undefined;
city: string | undefined;
region: string | undefined;
};
headers: Record<string, string | undefined>;
deviceId: string;
sessionId: string;
};
};
export const logsQueue = new Queue<LogsQueuePayload>(getQueueName('logs'), {
connection: getRedisQueue(),
defaultJobOptions: {
removeOnComplete: 100,
removeOnFail: 1000,
},
});

View File

@@ -8,7 +8,10 @@ describe('cachable', () => {
beforeEach(async () => { beforeEach(async () => {
redis = getRedisCache(); redis = getRedisCache();
// Clear any existing cache data for clean tests // Clear any existing cache data for clean tests
const keys = await redis.keys('cachable:*'); const keys = [
...await redis.keys('cachable:*'),
...await redis.keys('test-key*'),
];
if (keys.length > 0) { if (keys.length > 0) {
await redis.del(...keys); await redis.del(...keys);
} }
@@ -16,7 +19,10 @@ describe('cachable', () => {
afterEach(async () => { afterEach(async () => {
// Clean up after each test // Clean up after each test
const keys = await redis.keys('cachable:*'); const keys = [
...await redis.keys('cachable:*'),
...await redis.keys('test-key*'),
];
if (keys.length > 0) { if (keys.length > 0) {
await redis.del(...keys); await redis.del(...keys);
} }

View File

@@ -8,14 +8,14 @@
}, },
"dependencies": { "dependencies": {
"@openpanel/json": "workspace:*", "@openpanel/json": "workspace:*",
"ioredis": "5.8.2", "ioredis": "5.10.1",
"lru-cache": "^11.2.2" "lru-cache": "^11.2.7"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/db": "workspace:*", "@openpanel/db": "workspace:*",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -9,9 +9,9 @@
"react": "catalog:" "react": "catalog:"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "catalog:",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"prisma": "^5.1.1", "@types/react": "catalog:",
"prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -23,7 +23,7 @@
"@openpanel/web": "workspace:1.3.0-local" "@openpanel/web": "workspace:1.3.0-local"
}, },
"devDependencies": { "devDependencies": {
"astro": "^5.7.7" "astro": "^5.18.1"
}, },
"peerDependencies": { "peerDependencies": {
"astro": "^4.0.0 || ^5.0.0" "astro": "^4.0.0 || ^5.0.0"

View File

@@ -10,17 +10,17 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@openpanel/sdk": "workspace:1.3.0-local", "@openpanel/common": "workspace:*",
"@openpanel/common": "workspace:*" "@openpanel/sdk": "workspace:1.3.0-local"
}, },
"peerDependencies": { "peerDependencies": {
"express": "^4.17.0 || ^5.0.0" "express": "^4.17.0 || ^5.0.0"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/express": "^5.0.3", "@types/express": "^5.0.6",
"@types/request-ip": "^0.0.41", "@types/request-ip": "^0.0.41",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -20,7 +20,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/react": "catalog:", "@types/react": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -31,12 +31,12 @@
"nuxt": "^3.0.0 || ^4.0.0" "nuxt": "^3.0.0 || ^4.0.0"
}, },
"devDependencies": { "devDependencies": {
"@nuxt/kit": "^3.0.0", "@nuxt/kit": "^3.21.2",
"@nuxt/module-builder": "^1.0.2", "@nuxt/module-builder": "^1.0.2",
"@nuxt/types": "^2.18.1", "@nuxt/types": "^2.18.1",
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@vue/runtime-core": "^3.5.25", "@vue/runtime-core": "^3.5.31",
"typescript": "catalog:", "typescript": "catalog:",
"unbuild": "^3.6.1" "unbuild": "^3.6.1"
} }

View File

@@ -15,7 +15,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
}, },
"peerDependencies": { "peerDependencies": {

View File

@@ -9,12 +9,11 @@
"build": "rm -rf dist && tsup", "build": "rm -rf dist && tsup",
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": {},
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -7,6 +7,7 @@ import type {
IGroupPayload as GroupPayload, IGroupPayload as GroupPayload,
IIdentifyPayload as IdentifyPayload, IIdentifyPayload as IdentifyPayload,
IIncrementPayload as IncrementPayload, IIncrementPayload as IncrementPayload,
ISeverityText,
ITrackHandlerPayload as TrackHandlerPayload, ITrackHandlerPayload as TrackHandlerPayload,
ITrackPayload as TrackPayload, ITrackPayload as TrackPayload,
} from '@openpanel/validation'; } from '@openpanel/validation';
@@ -19,6 +20,7 @@ export type {
GroupPayload, GroupPayload,
IdentifyPayload, IdentifyPayload,
IncrementPayload, IncrementPayload,
ISeverityText,
TrackHandlerPayload, TrackHandlerPayload,
TrackPayload, TrackPayload,
}; };
@@ -29,6 +31,33 @@ export interface TrackProperties {
groups?: string[]; groups?: string[];
} }
export interface LogProperties {
/** Logger name (e.g. "com.example.MyActivity") */
loggerName?: string;
traceId?: string;
spanId?: string;
traceFlags?: number;
/** Log-level key-value attributes */
attributes?: Record<string, string>;
/** Resource/device attributes */
resource?: Record<string, string>;
/** ISO 8601 timestamp; defaults to now */
timestamp?: string;
}
interface LogPayloadForQueue {
body: string;
severity: ISeverityText;
loggerName?: string;
traceId?: string;
spanId?: string;
traceFlags?: number;
attributes?: Record<string, string>;
resource?: Record<string, string>;
timestamp: string;
profileId?: string;
}
export type UpsertGroupPayload = GroupPayload; export type UpsertGroupPayload = GroupPayload;
export interface OpenPanelOptions { export interface OpenPanelOptions {
@@ -57,6 +86,10 @@ export class OpenPanel {
sessionId?: string; sessionId?: string;
global?: Record<string, unknown>; global?: Record<string, unknown>;
queue: TrackHandlerPayload[] = []; queue: TrackHandlerPayload[] = [];
private logQueue: LogPayloadForQueue[] = [];
private logFlushTimer: ReturnType<typeof setTimeout> | null = null;
private logFlushIntervalMs = 2000;
private logFlushMaxSize = 50;
constructor(options: OpenPanelOptions) { constructor(options: OpenPanelOptions) {
this.options = options; this.options = options;
@@ -327,6 +360,67 @@ export class OpenPanel {
this.queue = remaining; this.queue = remaining;
} }
captureLog(
severity: ISeverityText,
body: string,
properties?: LogProperties,
) {
if (this.options.disabled) {
return;
}
const entry: LogPayloadForQueue = {
body,
severity,
timestamp: properties?.timestamp ?? new Date().toISOString(),
...(this.profileId ? { profileId: this.profileId } : {}),
...(properties?.loggerName ? { loggerName: properties.loggerName } : {}),
...(properties?.traceId ? { traceId: properties.traceId } : {}),
...(properties?.spanId ? { spanId: properties.spanId } : {}),
...(properties?.traceFlags !== undefined
? { traceFlags: properties.traceFlags }
: {}),
...(properties?.attributes ? { attributes: properties.attributes } : {}),
...(properties?.resource ? { resource: properties.resource } : {}),
};
this.logQueue.push(entry);
if (this.logQueue.length >= this.logFlushMaxSize) {
this.flushLogs();
return;
}
if (!this.logFlushTimer) {
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null;
this.flushLogs();
}, this.logFlushIntervalMs);
}
}
private async flushLogs() {
if (this.logFlushTimer) {
clearTimeout(this.logFlushTimer);
this.logFlushTimer = null;
}
if (this.logQueue.length === 0) {
return;
}
const batch = this.logQueue;
this.logQueue = [];
try {
await this.api.fetch('/logs', { logs: batch });
} catch (error) {
this.log('Failed to flush logs', error);
// Re-queue on failure
this.logQueue = batch.concat(this.logQueue);
}
}
log(...args: any[]) { log(...args: any[]) {
if (this.options.debug) { if (this.options.debug) {
console.log('[OpenPanel.dev]', ...args); console.log('[OpenPanel.dev]', ...args);

View File

@@ -17,7 +17,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"tsup": "^7.2.0", "tsup": "^8.5.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -1,3 +1,4 @@
export * from './src/root'; export * from './src/root';
export * from './src/trpc'; export * from './src/trpc';
export { getProjectAccess } from './src/access'; export { getProjectAccess } from './src/access';
export type { IServiceLog } from './src/routers/log';

View File

@@ -16,17 +16,17 @@
"@openpanel/integrations": "workspace:^", "@openpanel/integrations": "workspace:^",
"@openpanel/js-runtime": "workspace:*", "@openpanel/js-runtime": "workspace:*",
"@openpanel/payments": "workspace:^", "@openpanel/payments": "workspace:^",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*", "@openpanel/redis": "workspace:*",
"@openpanel/validation": "workspace:*", "@openpanel/validation": "workspace:*",
"@openpanel/queue": "workspace:*",
"@trpc-limiter/redis": "^0.0.2", "@trpc-limiter/redis": "^0.0.2",
"@trpc/client": "^11.6.0", "@trpc/client": "^11.16.0",
"@trpc/server": "^11.6.0", "@trpc/server": "^11.16.0",
"date-fns": "^3.3.1", "date-fns": "^3.6.0",
"mathjs": "^12.3.2", "mathjs": "^15.1.1",
"prisma-error-enum": "^0.1.3", "prisma-error-enum": "^0.1.3",
"ramda": "^0.29.1", "ramda": "^0.32.0",
"short-unique-id": "^5.0.3", "short-unique-id": "^5.3.2",
"sqlstring": "^2.3.3", "sqlstring": "^2.3.3",
"superjson": "^1.13.3", "superjson": "^1.13.3",
"uuid": "^9.0.1", "uuid": "^9.0.1",
@@ -35,9 +35,9 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/ramda": "^0.29.6", "@types/ramda": "^0.31.1",
"@types/sqlstring": "^2.3.2", "@types/sqlstring": "^2.3.2",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -8,6 +8,7 @@ import { eventRouter } from './routers/event';
import { groupRouter } from './routers/group'; import { groupRouter } from './routers/group';
import { gscRouter } from './routers/gsc'; import { gscRouter } from './routers/gsc';
import { importRouter } from './routers/import'; import { importRouter } from './routers/import';
import { logRouter } from './routers/log';
import { insightRouter } from './routers/insight'; import { insightRouter } from './routers/insight';
import { integrationRouter } from './routers/integration'; import { integrationRouter } from './routers/integration';
import { notificationRouter } from './routers/notification'; import { notificationRouter } from './routers/notification';
@@ -57,6 +58,7 @@ export const appRouter = createTRPCRouter({
email: emailRouter, email: emailRouter,
gsc: gscRouter, gsc: gscRouter,
group: groupRouter, group: groupRouter,
log: logRouter,
}); });
// export type definition of API // export type definition of API

View File

@@ -0,0 +1,212 @@
import { chQuery, convertClickhouseDateToJs } from '@openpanel/db';
import { zSeverityText } from '@openpanel/validation';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../trpc';
export interface IServiceLog {
id: string;
projectId: string;
deviceId: string;
profileId: string;
sessionId: string;
timestamp: Date;
severityNumber: number;
severityText: string;
body: string;
traceId: string;
spanId: string;
traceFlags: number;
loggerName: string;
attributes: Record<string, string>;
resource: Record<string, string>;
sdkName: string;
sdkVersion: string;
country: string;
city: string;
region: string;
os: string;
osVersion: string;
browser: string;
browserVersion: string;
device: string;
brand: string;
model: string;
}
interface IClickhouseLog {
id: string;
project_id: string;
device_id: string;
profile_id: string;
session_id: string;
timestamp: string;
severity_number: number;
severity_text: string;
body: string;
trace_id: string;
span_id: string;
trace_flags: number;
logger_name: string;
attributes: Record<string, string>;
resource: Record<string, string>;
sdk_name: string;
sdk_version: string;
country: string;
city: string;
region: string;
os: string;
os_version: string;
browser: string;
browser_version: string;
device: string;
brand: string;
model: string;
}
function toServiceLog(row: IClickhouseLog): IServiceLog {
return {
id: row.id,
projectId: row.project_id,
deviceId: row.device_id,
profileId: row.profile_id,
sessionId: row.session_id,
timestamp: convertClickhouseDateToJs(row.timestamp),
severityNumber: row.severity_number,
severityText: row.severity_text,
body: row.body,
traceId: row.trace_id,
spanId: row.span_id,
traceFlags: row.trace_flags,
loggerName: row.logger_name,
attributes: row.attributes,
resource: row.resource,
sdkName: row.sdk_name,
sdkVersion: row.sdk_version,
country: row.country,
city: row.city,
region: row.region,
os: row.os,
osVersion: row.os_version,
browser: row.browser,
browserVersion: row.browser_version,
device: row.device,
brand: row.brand,
model: row.model,
};
}
export const logRouter = createTRPCRouter({
list: protectedProcedure
.input(
z.object({
projectId: z.string(),
cursor: z.string().nullish(),
severity: z.array(zSeverityText).optional(),
search: z.string().optional(),
loggerName: z.string().optional(),
startDate: z.date().optional(),
endDate: z.date().optional(),
take: z.number().default(50),
}),
)
.query(async ({ input }) => {
const { projectId, cursor, severity, search, loggerName, startDate, endDate, take } = input;
const conditions: string[] = [
`project_id = ${sqlstring.escape(projectId)}`,
];
if (cursor) {
conditions.push(`timestamp < ${sqlstring.escape(cursor)}`);
}
if (severity && severity.length > 0) {
const escaped = severity.map((s) => sqlstring.escape(s)).join(', ');
conditions.push(`severity_text IN (${escaped})`);
}
if (search) {
conditions.push(`body ILIKE ${sqlstring.escape(`%${search}%`)}`);
}
if (loggerName) {
conditions.push(`logger_name = ${sqlstring.escape(loggerName)}`);
}
if (startDate) {
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
}
if (endDate) {
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
}
const where = conditions.join(' AND ');
const rows = await chQuery<IClickhouseLog>(
`SELECT
id, project_id, device_id, profile_id, session_id,
timestamp, severity_number, severity_text, body,
trace_id, span_id, trace_flags, logger_name,
attributes, resource,
sdk_name, sdk_version,
country, city, region, os, os_version,
browser, browser_version, device, brand, model
FROM logs
WHERE ${where}
ORDER BY timestamp DESC
LIMIT ${take + 1}`,
);
const hasMore = rows.length > take;
const data = rows.slice(0, take).map(toServiceLog);
const lastItem = data[data.length - 1];
return {
data,
meta: {
next: hasMore && lastItem ? lastItem.timestamp.toISOString() : null,
},
};
}),
severityCounts: protectedProcedure
.input(
z.object({
projectId: z.string(),
startDate: z.date().optional(),
endDate: z.date().optional(),
}),
)
.query(async ({ input }) => {
const { projectId, startDate, endDate } = input;
const conditions: string[] = [
`project_id = ${sqlstring.escape(projectId)}`,
];
if (startDate) {
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
}
if (endDate) {
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
}
const where = conditions.join(' AND ');
const rows = await chQuery<{ severity_text: string; count: number }>(
`SELECT severity_text, count() AS count
FROM logs
WHERE ${where}
GROUP BY severity_text
ORDER BY count DESC`,
);
return rows.reduce<Record<string, number>>((acc, row) => {
acc[row.severity_text] = row.count;
return acc;
}, {});
}),
});

View File

@@ -13,7 +13,7 @@
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"prisma": "^5.1.1", "prisma": "^5.22.0",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -626,3 +626,4 @@ export type ICreateImport = z.infer<typeof zCreateImport>;
export * from './types.insights'; export * from './types.insights';
export * from './track.validation'; export * from './track.validation';
export * from './event-blocklist'; export * from './event-blocklist';
export * from './log.validation';

View File

@@ -0,0 +1,60 @@
import { z } from 'zod';
/**
* OTel severity number mapping (subset):
* TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21
*/
export const SEVERITY_TEXT_TO_NUMBER: Record<string, number> = {
trace: 1,
debug: 5,
info: 9,
warn: 13,
warning: 13,
error: 17,
fatal: 21,
critical: 21,
};
export const zSeverityText = z.enum([
'trace',
'debug',
'info',
'warn',
'warning',
'error',
'fatal',
'critical',
]);
export type ISeverityText = z.infer<typeof zSeverityText>;
export const zLogPayload = z.object({
/** Log message / body */
body: z.string().min(1),
/** Severity level as text */
severity: zSeverityText.default('info'),
/** Optional override for the numeric OTel severity (1-24) */
severityNumber: z.number().int().min(1).max(24).optional(),
/** ISO 8601 timestamp; defaults to server receive time if omitted */
timestamp: z.string().datetime({ offset: true }).optional(),
/** Logger name (e.g. "com.example.MyActivity") */
loggerName: z.string().optional(),
/** W3C trace context */
traceId: z.string().optional(),
spanId: z.string().optional(),
traceFlags: z.number().int().min(0).optional(),
/** Log-level key-value attributes */
attributes: z.record(z.string(), z.string()).optional(),
/** Resource/device attributes (app version, runtime, etc.) */
resource: z.record(z.string(), z.string()).optional(),
/** Profile/user ID to associate with this log */
profileId: z.union([z.string().min(1), z.number()]).optional(),
});
export type ILogPayload = z.infer<typeof zLogPayload>;
export const zLogBatchPayload = z.object({
logs: z.array(zLogPayload).min(1).max(500),
});
export type ILogBatchPayload = z.infer<typeof zLogBatchPayload>;

View File

@@ -0,0 +1,42 @@
diff --git a/.npmignore b/.npmignore
deleted file mode 100644
index 34e4f5c298de294fa5c1c1769b6489eb047bde9a..0000000000000000000000000000000000000000
diff --git a/index.js b/index.js
index 5462c1f830bdbe79bf2b1fcfd811cd9799b4dd11..689421d49e83d168981a0c7d1fef59a0b0e56963 100644
--- a/index.js
+++ b/index.js
@@ -3,6 +3,9 @@
var Buffer = require('buffer').Buffer; // browserify
var SlowBuffer = require('buffer').SlowBuffer;
+// Handle Node.js v25+ where SlowBuffer was removed
+var hasSlowBuffer = !!SlowBuffer;
+
module.exports = bufferEq;
function bufferEq(a, b) {
@@ -28,14 +31,18 @@ function bufferEq(a, b) {
}
bufferEq.install = function() {
- Buffer.prototype.equal = SlowBuffer.prototype.equal = function equal(that) {
- return bufferEq(this, that);
- };
+ Buffer.prototype.equal = function equal(that) {
+ return bufferEq(this, that);
+ };
+ if (hasSlowBuffer) {
+ SlowBuffer.prototype.equal = Buffer.prototype.equal;
+ }
};
var origBufEqual = Buffer.prototype.equal;
-var origSlowBufEqual = SlowBuffer.prototype.equal;
bufferEq.restore = function() {
- Buffer.prototype.equal = origBufEqual;
- SlowBuffer.prototype.equal = origSlowBufEqual;
+ Buffer.prototype.equal = origBufEqual;
+ if (hasSlowBuffer && SlowBuffer.prototype) {
+ delete SlowBuffer.prototype.equal;
+ }
};

28713
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -7,10 +7,10 @@ packages:
# Define a catalog of version ranges. # Define a catalog of version ranges.
catalog: catalog:
zod: ^3.24.2 zod: ^3.24.2
react: ^19.2.3 react: ^19.2.4
"@types/react": ^19.2.3 "@types/react": ^19.2.14
"react-dom": ^19.2.3 "react-dom": ^19.2.4
"@types/react-dom": ^19.2.3 "@types/react-dom": ^19.2.3
"@types/node": ^24.7.1 "@types/node": ^25.5.0
typescript: ^5.9.3 typescript: ^5.9.3
groupmq: 2.0.0-next.1 groupmq: 2.0.0-next.1

View File

@@ -23,7 +23,7 @@ services:
max-file: "3" max-file: "3"
op-db: op-db:
image: postgres:14-alpine image: postgres:18.3-alpine
restart: always restart: always
volumes: volumes:
- op-db-data:/var/lib/postgresql/data - op-db-data:/var/lib/postgresql/data
@@ -45,7 +45,7 @@ services:
# - 5432:5432 # - 5432:5432
op-kv: op-kv:
image: redis:7.2.5-alpine image: redis:8.6.2-alpine
restart: always restart: always
volumes: volumes:
- op-kv-data:/data - op-kv-data:/data
@@ -65,7 +65,7 @@ services:
# - 6379:6379 # - 6379:6379
op-ch: op-ch:
image: clickhouse/clickhouse-server:25.10.2.65 image: clickhouse/clickhouse-server:26.3.2.3
restart: always restart: always
volumes: volumes:
- op-ch-data:/var/lib/clickhouse - op-ch-data:/var/lib/clickhouse

View File

@@ -9,12 +9,12 @@
}, },
"dependencies": { "dependencies": {
"arg": "^5.0.2", "arg": "^5.0.2",
"semver": "^7.5.4" "semver": "^7.7.4"
}, },
"devDependencies": { "devDependencies": {
"@openpanel/tsconfig": "workspace:*", "@openpanel/tsconfig": "workspace:*",
"@types/node": "catalog:", "@types/node": "catalog:",
"@types/semver": "^7.5.4", "@types/semver": "^7.7.1",
"typescript": "catalog:" "typescript": "catalog:"
} }
} }

View File

@@ -3,8 +3,12 @@
"version": "0.1.0", "version": "0.1.0",
"type": "module", "type": "module",
"private": true, "private": true,
"files": ["base.json", "sdk.json", "tsup.config.json"], "files": [
"base.json",
"sdk.json",
"tsup.config.json"
],
"devDependencies": { "devDependencies": {
"tsup": "^7.2.0" "tsup": "^8.5.1"
} }
} }