Compare commits
4 Commits
test-profi
...
be64494aa9
| Author | SHA1 | Date | |
|---|---|---|---|
|
be64494aa9
|
|||
|
eefbeac7f8
|
|||
|
0672857974
|
|||
|
|
a1ce71ffb6 |
55
.gitea/workflows/docker-build-api.yml
Normal file
55
.gitea/workflows/docker-build-api.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
name: Build and Push API
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.zias.be
|
||||
OWNER: zias
|
||||
|
||||
jobs:
|
||||
build-api:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ env.OWNER }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=sha-,format=short
|
||||
type=semver,pattern={{version}}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/api/Dockerfile
|
||||
target: runner
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api:buildcache
|
||||
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-api:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
|
||||
build-args: |
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres
|
||||
53
.gitea/workflows/docker-build-dashboard.yml
Normal file
53
.gitea/workflows/docker-build-dashboard.yml
Normal file
@@ -0,0 +1,53 @@
|
||||
name: Build and Push Dashboard
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.zias.be
|
||||
OWNER: zias
|
||||
|
||||
jobs:
|
||||
build-dashboard:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ env.OWNER }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=sha-,format=short
|
||||
type=semver,pattern={{version}}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/start/Dockerfile
|
||||
target: runner
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard:buildcache
|
||||
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-dashboard:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
|
||||
55
.gitea/workflows/docker-build-worker.yml
Normal file
55
.gitea/workflows/docker-build-worker.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
name: Build and Push Worker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.zias.be
|
||||
OWNER: zias
|
||||
|
||||
jobs:
|
||||
build-worker:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ env.OWNER }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=sha-,format=short
|
||||
type=semver,pattern={{version}}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/worker/Dockerfile
|
||||
target: runner
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker:buildcache
|
||||
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-worker:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
|
||||
build-args: |
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres
|
||||
@@ -10,14 +10,14 @@
|
||||
"dependencies": {
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/db": "workspace:*",
|
||||
"chalk": "^5.3.0",
|
||||
"chalk": "^5.6.2",
|
||||
"fuzzy": "^0.1.3",
|
||||
"inquirer": "^9.3.5",
|
||||
"inquirer": "^9.3.8",
|
||||
"inquirer-autocomplete-prompt": "^3.0.1",
|
||||
"jiti": "^2.4.2"
|
||||
"jiti": "^2.6.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/inquirer": "^9.0.7",
|
||||
"@types/inquirer": "^9.0.9",
|
||||
"@types/inquirer-autocomplete-prompt": "^3.0.3",
|
||||
"@types/node": "catalog:",
|
||||
"typescript": "catalog:"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG NODE_VERSION=22.20.0
|
||||
ARG NODE_VERSION=22.22.2
|
||||
|
||||
FROM node:${NODE_VERSION}-slim AS base
|
||||
|
||||
|
||||
@@ -12,11 +12,11 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^1.2.10",
|
||||
"@ai-sdk/openai": "^1.3.12",
|
||||
"@fastify/compress": "^8.1.0",
|
||||
"@ai-sdk/anthropic": "^1.2.12",
|
||||
"@ai-sdk/openai": "^1.3.24",
|
||||
"@fastify/compress": "^8.3.1",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/cors": "^11.1.0",
|
||||
"@fastify/cors": "^11.2.0",
|
||||
"@fastify/rate-limit": "^10.3.0",
|
||||
"@fastify/websocket": "^11.2.0",
|
||||
"@node-rs/argon2": "^2.0.2",
|
||||
@@ -33,36 +33,36 @@
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/trpc": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@trpc/server": "^11.6.0",
|
||||
"ai": "^4.2.10",
|
||||
"@trpc/server": "^11.16.0",
|
||||
"ai": "^4.3.19",
|
||||
"fast-json-stable-hash": "^1.0.3",
|
||||
"fastify": "^5.6.1",
|
||||
"fastify": "^5.8.4",
|
||||
"fastify-metrics": "^12.1.0",
|
||||
"fastify-raw-body": "^5.0.0",
|
||||
"groupmq": "catalog:",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"ramda": "^0.29.1",
|
||||
"sharp": "^0.33.5",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"ramda": "^0.32.0",
|
||||
"sharp": "^0.34.5",
|
||||
"source-map-support": "^0.5.21",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"svix": "^1.24.0",
|
||||
"url-metadata": "^5.4.1",
|
||||
"svix": "^1.89.0",
|
||||
"url-metadata": "^5.4.3",
|
||||
"uuid": "^9.0.1",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@faker-js/faker": "^9.0.1",
|
||||
"@faker-js/faker": "^9.9.0",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
"@types/jsonwebtoken": "^9.0.9",
|
||||
"@types/ramda": "^0.30.2",
|
||||
"@types/jsonwebtoken": "^9.0.10",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/source-map-support": "^0.5.10",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"@types/ws": "^8.5.14",
|
||||
"js-yaml": "^4.1.0",
|
||||
"tsdown": "0.14.2",
|
||||
"@types/uuid": "^11.0.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"js-yaml": "^4.1.1",
|
||||
"tsdown": "0.21.7",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,7 @@
|
||||
import type { WebSocket } from '@fastify/websocket';
|
||||
import { eventBuffer } from '@openpanel/db';
|
||||
import { setSuperJson } from '@openpanel/json';
|
||||
import {
|
||||
psubscribeToPublishedEvent,
|
||||
subscribeToPublishedEvent,
|
||||
} from '@openpanel/redis';
|
||||
import { subscribeToPublishedEvent } from '@openpanel/redis';
|
||||
import { getProjectAccess } from '@openpanel/trpc';
|
||||
import { getOrganizationAccess } from '@openpanel/trpc/src/access';
|
||||
import type { FastifyRequest } from 'fastify';
|
||||
@@ -39,19 +36,8 @@ export function wsVisitors(
|
||||
}
|
||||
);
|
||||
|
||||
const punsubscribe = psubscribeToPublishedEvent(
|
||||
'__keyevent@0__:expired',
|
||||
(key) => {
|
||||
const [, , projectId] = key.split(':');
|
||||
if (projectId === params.projectId) {
|
||||
sendCount();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
socket.on('close', () => {
|
||||
unsubscribe();
|
||||
punsubscribe();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
68
apps/api/src/controllers/logs.controller.ts
Normal file
68
apps/api/src/controllers/logs.controller.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { parseUserAgent } from '@openpanel/common/server';
|
||||
import { getSalts } from '@openpanel/db';
|
||||
import { getGeoLocation } from '@openpanel/geo';
|
||||
import { type LogsQueuePayload, logsQueue } from '@openpanel/queue';
|
||||
import { type ILogBatchPayload, zLogBatchPayload } from '@openpanel/validation';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { getDeviceId } from '@/utils/ids';
|
||||
import { getStringHeaders } from './track.controller';
|
||||
|
||||
export async function handler(
|
||||
request: FastifyRequest<{ Body: ILogBatchPayload }>,
|
||||
reply: FastifyReply,
|
||||
) {
|
||||
const projectId = request.client?.projectId;
|
||||
if (!projectId) {
|
||||
return reply.status(400).send({ status: 400, error: 'Missing projectId' });
|
||||
}
|
||||
|
||||
const validationResult = zLogBatchPayload.safeParse(request.body);
|
||||
if (!validationResult.success) {
|
||||
return reply.status(400).send({
|
||||
status: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Validation failed',
|
||||
errors: validationResult.error.errors,
|
||||
});
|
||||
}
|
||||
|
||||
const { logs } = validationResult.data;
|
||||
|
||||
const ip = request.clientIp;
|
||||
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
|
||||
const headers = getStringHeaders(request.headers);
|
||||
const receivedAt = new Date().toISOString();
|
||||
|
||||
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
|
||||
const { deviceId, sessionId } = await getDeviceId({ projectId, ip, ua, salts });
|
||||
const uaInfo = parseUserAgent(ua, undefined);
|
||||
|
||||
const jobs: LogsQueuePayload[] = logs.map((log) => ({
|
||||
type: 'incomingLog' as const,
|
||||
payload: {
|
||||
projectId,
|
||||
log: {
|
||||
...log,
|
||||
timestamp: log.timestamp ?? receivedAt,
|
||||
},
|
||||
uaInfo,
|
||||
geo: {
|
||||
country: geo.country,
|
||||
city: geo.city,
|
||||
region: geo.region,
|
||||
},
|
||||
headers,
|
||||
deviceId,
|
||||
sessionId,
|
||||
},
|
||||
}));
|
||||
|
||||
await logsQueue.addBulk(
|
||||
jobs.map((job) => ({
|
||||
name: 'incomingLog',
|
||||
data: job,
|
||||
})),
|
||||
);
|
||||
|
||||
return reply.status(200).send({ ok: true, count: logs.length });
|
||||
}
|
||||
@@ -44,6 +44,7 @@ import manageRouter from './routes/manage.router';
|
||||
import miscRouter from './routes/misc.router';
|
||||
import oauthRouter from './routes/oauth-callback.router';
|
||||
import profileRouter from './routes/profile.router';
|
||||
import logsRouter from './routes/logs.router';
|
||||
import trackRouter from './routes/track.router';
|
||||
import webhookRouter from './routes/webhook.router';
|
||||
import { HttpError } from './utils/errors';
|
||||
@@ -209,6 +210,7 @@ const startServer = async () => {
|
||||
instance.register(importRouter, { prefix: '/import' });
|
||||
instance.register(insightsRouter, { prefix: '/insights' });
|
||||
instance.register(trackRouter, { prefix: '/track' });
|
||||
instance.register(logsRouter, { prefix: '/logs' });
|
||||
instance.register(manageRouter, { prefix: '/manage' });
|
||||
// Keep existing endpoints for backward compatibility
|
||||
instance.get('/healthcheck', healthcheck);
|
||||
|
||||
17
apps/api/src/routes/logs.router.ts
Normal file
17
apps/api/src/routes/logs.router.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import type { FastifyPluginCallback } from 'fastify';
|
||||
import { handler } from '@/controllers/logs.controller';
|
||||
import { clientHook } from '@/hooks/client.hook';
|
||||
import { duplicateHook } from '@/hooks/duplicate.hook';
|
||||
|
||||
const logsRouter: FastifyPluginCallback = async (fastify) => {
|
||||
fastify.addHook('preValidation', duplicateHook);
|
||||
fastify.addHook('preHandler', clientHook);
|
||||
|
||||
fastify.route({
|
||||
method: 'POST',
|
||||
url: '/',
|
||||
handler,
|
||||
});
|
||||
};
|
||||
|
||||
export default logsRouter;
|
||||
@@ -16,11 +16,11 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@nivo/funnel": "^0.99.0",
|
||||
"@number-flow/react": "0.5.10",
|
||||
"@opennextjs/cloudflare": "^1.17.1",
|
||||
"@number-flow/react": "0.6.0",
|
||||
"@opennextjs/cloudflare": "^1.18.0",
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/geo": "workspace:*",
|
||||
"@openpanel/nextjs": "^1.2.0",
|
||||
"@openpanel/nextjs": "^1.4.0",
|
||||
"@openpanel/payments": "workspace:^",
|
||||
"@openpanel/sdk-info": "workspace:^",
|
||||
"@openstatus/react": "0.0.3",
|
||||
@@ -28,37 +28,37 @@
|
||||
"@radix-ui/react-slider": "1.3.6",
|
||||
"@radix-ui/react-slot": "1.2.4",
|
||||
"@radix-ui/react-tooltip": "1.2.8",
|
||||
"cheerio": "^1.0.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"class-variance-authority": "0.7.1",
|
||||
"clsx": "2.1.1",
|
||||
"dotted-map": "2.2.3",
|
||||
"framer-motion": "12.23.25",
|
||||
"fumadocs-core": "16.2.2",
|
||||
"fumadocs-mdx": "14.0.4",
|
||||
"fumadocs-ui": "16.2.2",
|
||||
"geist": "1.5.1",
|
||||
"dotted-map": "3.1.0",
|
||||
"framer-motion": "12.38.0",
|
||||
"fumadocs-core": "16.7.7",
|
||||
"fumadocs-mdx": "14.2.11",
|
||||
"fumadocs-ui": "16.7.7",
|
||||
"geist": "1.7.0",
|
||||
"lucide-react": "^0.555.0",
|
||||
"next": "16.0.7",
|
||||
"next": "16.2.1",
|
||||
"next-themes": "^0.4.6",
|
||||
"react": "catalog:",
|
||||
"react-dom": "catalog:",
|
||||
"react-markdown": "^10.1.0",
|
||||
"recharts": "^2.15.0",
|
||||
"recharts": "^2.15.4",
|
||||
"rehype-external-links": "3.0.0",
|
||||
"tailwind-merge": "3.4.0",
|
||||
"tailwind-merge": "3.5.0",
|
||||
"tailwindcss-animate": "1.0.7",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4.1.17",
|
||||
"@tailwindcss/postcss": "^4.2.2",
|
||||
"@types/mdx": "^2.0.13",
|
||||
"@types/node": "catalog:",
|
||||
"@types/react": "catalog:",
|
||||
"@types/react-dom": "catalog:",
|
||||
"autoprefixer": "^10.4.22",
|
||||
"postcss": "^8.5.6",
|
||||
"tailwindcss": "4.1.17",
|
||||
"autoprefixer": "^10.4.27",
|
||||
"postcss": "^8.5.8",
|
||||
"tailwindcss": "4.2.2",
|
||||
"typescript": "catalog:",
|
||||
"wrangler": "^4.65.0"
|
||||
"wrangler": "^4.78.0"
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG NODE_VERSION=22.20.0
|
||||
ARG NODE_VERSION=22.22.2
|
||||
|
||||
FROM node:${NODE_VERSION}-slim AS base
|
||||
|
||||
|
||||
@@ -17,21 +17,21 @@
|
||||
"with-env": "dotenv -e ../../.env -c --"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/react": "^1.2.5",
|
||||
"@codemirror/commands": "^6.7.0",
|
||||
"@codemirror/lang-javascript": "^6.2.0",
|
||||
"@codemirror/lang-json": "^6.0.1",
|
||||
"@codemirror/state": "^6.4.0",
|
||||
"@ai-sdk/react": "^1.2.12",
|
||||
"@codemirror/commands": "^6.10.3",
|
||||
"@codemirror/lang-javascript": "^6.2.5",
|
||||
"@codemirror/lang-json": "^6.0.2",
|
||||
"@codemirror/state": "^6.6.0",
|
||||
"@codemirror/theme-one-dark": "^6.1.3",
|
||||
"@codemirror/view": "^6.35.0",
|
||||
"@codemirror/view": "^6.40.0",
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
"@dnd-kit/sortable": "^10.0.0",
|
||||
"@dnd-kit/utilities": "^3.2.2",
|
||||
"@faker-js/faker": "^9.6.0",
|
||||
"@hookform/resolvers": "^3.3.4",
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"@faker-js/faker": "^9.9.0",
|
||||
"@hookform/resolvers": "^3.10.0",
|
||||
"@hyperdx/node-opentelemetry": "^0.10.3",
|
||||
"@nivo/sankey": "^0.99.0",
|
||||
"@number-flow/react": "0.5.10",
|
||||
"@number-flow/react": "0.6.0",
|
||||
"@openpanel/common": "workspace:^",
|
||||
"@openpanel/constants": "workspace:^",
|
||||
"@openpanel/importer": "workspace:^",
|
||||
@@ -40,100 +40,100 @@
|
||||
"@openpanel/payments": "workspace:*",
|
||||
"@openpanel/sdk-info": "workspace:^",
|
||||
"@openpanel/validation": "workspace:^",
|
||||
"@openpanel/web": "^1.0.1",
|
||||
"@openpanel/web": "^1.3.0",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-alert-dialog": "^1.1.15",
|
||||
"@radix-ui/react-aspect-ratio": "^1.1.7",
|
||||
"@radix-ui/react-avatar": "^1.1.10",
|
||||
"@radix-ui/react-aspect-ratio": "^1.1.8",
|
||||
"@radix-ui/react-avatar": "^1.1.11",
|
||||
"@radix-ui/react-checkbox": "^1.3.3",
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
"@radix-ui/react-dropdown-menu": "^2.1.16",
|
||||
"@radix-ui/react-label": "^2.1.7",
|
||||
"@radix-ui/react-label": "^2.1.8",
|
||||
"@radix-ui/react-popover": "^1.1.15",
|
||||
"@radix-ui/react-portal": "^1.1.9",
|
||||
"@radix-ui/react-progress": "^1.1.7",
|
||||
"@radix-ui/react-portal": "^1.1.10",
|
||||
"@radix-ui/react-progress": "^1.1.8",
|
||||
"@radix-ui/react-radio-group": "^1.3.8",
|
||||
"@radix-ui/react-scroll-area": "^1.2.10",
|
||||
"@radix-ui/react-select": "^2.2.6",
|
||||
"@radix-ui/react-separator": "^1.1.7",
|
||||
"@radix-ui/react-slider": "1.2.3",
|
||||
"@radix-ui/react-slot": "^1.2.3",
|
||||
"@radix-ui/react-separator": "^1.1.8",
|
||||
"@radix-ui/react-slider": "1.3.6",
|
||||
"@radix-ui/react-slot": "^1.2.4",
|
||||
"@radix-ui/react-switch": "^1.2.6",
|
||||
"@radix-ui/react-tabs": "^1.1.13",
|
||||
"@radix-ui/react-toast": "^1.2.15",
|
||||
"@radix-ui/react-toggle": "^1.1.10",
|
||||
"@radix-ui/react-toggle-group": "^1.1.11",
|
||||
"@radix-ui/react-tooltip": "^1.2.8",
|
||||
"@reduxjs/toolkit": "^2.8.2",
|
||||
"@sentry/tanstackstart-react": "^9.12.0",
|
||||
"@reduxjs/toolkit": "^2.11.2",
|
||||
"@sentry/tanstackstart-react": "^10.46.0",
|
||||
"@tailwindcss/container-queries": "^0.1.1",
|
||||
"@tailwindcss/typography": "^0.5.15",
|
||||
"@tailwindcss/vite": "^4.0.6",
|
||||
"@tailwindcss/typography": "^0.5.19",
|
||||
"@tailwindcss/vite": "^4.2.2",
|
||||
"@tanstack/match-sorter-utils": "^8.19.4",
|
||||
"@tanstack/nitro-v2-vite-plugin": "^1.133.19",
|
||||
"@tanstack/react-devtools": "^0.7.6",
|
||||
"@tanstack/react-query": "^5.90.2",
|
||||
"@tanstack/react-query-devtools": "^5.90.2",
|
||||
"@tanstack/react-router": "^1.132.47",
|
||||
"@tanstack/react-router-devtools": "^1.132.51",
|
||||
"@tanstack/react-router-ssr-query": "^1.132.47",
|
||||
"@tanstack/react-start": "^1.132.56",
|
||||
"@tanstack/react-store": "^0.8.0",
|
||||
"@tanstack/nitro-v2-vite-plugin": "^1.154.9",
|
||||
"@tanstack/react-devtools": "^0.10.0",
|
||||
"@tanstack/react-query": "^5.95.2",
|
||||
"@tanstack/react-query-devtools": "^5.95.2",
|
||||
"@tanstack/react-router": "^1.168.10",
|
||||
"@tanstack/react-router-devtools": "^1.166.11",
|
||||
"@tanstack/react-router-ssr-query": "^1.166.10",
|
||||
"@tanstack/react-start": "^1.167.16",
|
||||
"@tanstack/react-store": "^0.9.3",
|
||||
"@tanstack/react-table": "^8.21.3",
|
||||
"@tanstack/react-virtual": "^3.13.12",
|
||||
"@tanstack/router-plugin": "^1.132.56",
|
||||
"@tanstack/store": "^0.8.0",
|
||||
"@trpc/client": "^11.6.0",
|
||||
"@trpc/react-query": "^11.6.0",
|
||||
"@trpc/server": "^11.6.0",
|
||||
"@trpc/tanstack-react-query": "^11.6.0",
|
||||
"@tanstack/react-virtual": "^3.13.23",
|
||||
"@tanstack/router-plugin": "^1.167.12",
|
||||
"@tanstack/store": "^0.9.3",
|
||||
"@trpc/client": "^11.16.0",
|
||||
"@trpc/react-query": "^11.16.0",
|
||||
"@trpc/server": "^11.16.0",
|
||||
"@trpc/tanstack-react-query": "^11.16.0",
|
||||
"@types/d3": "^7.4.3",
|
||||
"ai": "^4.2.10",
|
||||
"ai": "^4.3.19",
|
||||
"bind-event-listener": "^3.0.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^0.2.1",
|
||||
"codemirror": "^6.0.1",
|
||||
"d3": "^7.8.5",
|
||||
"date-fns": "^3.3.1",
|
||||
"debounce": "^2.2.0",
|
||||
"codemirror": "^6.0.2",
|
||||
"d3": "^7.9.0",
|
||||
"date-fns": "^3.6.0",
|
||||
"debounce": "^3.0.0",
|
||||
"embla-carousel-autoplay": "^8.6.0",
|
||||
"embla-carousel-react": "8.0.0-rc22",
|
||||
"flag-icons": "^7.1.0",
|
||||
"framer-motion": "^11.0.28",
|
||||
"hamburger-react": "^2.5.0",
|
||||
"input-otp": "^1.2.4",
|
||||
"javascript-time-ago": "^2.5.9",
|
||||
"katex": "^0.16.21",
|
||||
"embla-carousel-react": "8.6.0",
|
||||
"flag-icons": "^7.5.0",
|
||||
"framer-motion": "^12.38.0",
|
||||
"hamburger-react": "^2.5.2",
|
||||
"input-otp": "^1.4.2",
|
||||
"javascript-time-ago": "^2.6.4",
|
||||
"katex": "^0.16.44",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"lodash.isequal": "^4.5.0",
|
||||
"lodash.throttle": "^4.1.1",
|
||||
"lottie-react": "^2.4.0",
|
||||
"lottie-react": "^2.4.1",
|
||||
"lucide-react": "^0.476.0",
|
||||
"mitt": "^3.0.1",
|
||||
"nuqs": "^2.5.2",
|
||||
"nuqs": "^2.8.9",
|
||||
"prisma-error-enum": "^0.1.3",
|
||||
"pushmodal": "^1.0.3",
|
||||
"ramda": "^0.29.1",
|
||||
"pushmodal": "^1.0.5",
|
||||
"ramda": "^0.32.0",
|
||||
"random-animal-name": "^0.1.1",
|
||||
"rc-virtual-list": "^3.14.5",
|
||||
"rc-virtual-list": "^3.19.2",
|
||||
"react": "catalog:",
|
||||
"react-animate-height": "^3.2.3",
|
||||
"react-animated-numbers": "^1.1.1",
|
||||
"react-day-picker": "^9.9.0",
|
||||
"react-day-picker": "^9.14.0",
|
||||
"react-dom": "catalog:",
|
||||
"react-grid-layout": "^1.5.2",
|
||||
"react-hook-form": "^7.50.1",
|
||||
"react-in-viewport": "1.0.0-beta.8",
|
||||
"react-grid-layout": "^1.5.3",
|
||||
"react-hook-form": "^7.72.0",
|
||||
"react-in-viewport": "1.0.0-beta.9",
|
||||
"react-markdown": "^10.1.0",
|
||||
"react-redux": "^8.1.3",
|
||||
"react-resizable": "^3.0.5",
|
||||
"react-responsive": "^9.0.2",
|
||||
"react-resizable": "^3.1.3",
|
||||
"react-responsive": "^10.0.1",
|
||||
"react-simple-maps": "3.0.0",
|
||||
"react-svg-worldmap": "2.0.0-alpha.16",
|
||||
"react-syntax-highlighter": "^15.5.0",
|
||||
"react-use-websocket": "^4.7.0",
|
||||
"react-virtualized-auto-sizer": "^1.0.22",
|
||||
"react-svg-worldmap": "2.0.1",
|
||||
"react-syntax-highlighter": "^16.1.1",
|
||||
"react-use-websocket": "^4.13.0",
|
||||
"react-virtualized-auto-sizer": "^1.0.26",
|
||||
"recharts": "^2.15.4",
|
||||
"rehype-katex": "^7.0.1",
|
||||
"remark-gfm": "^4.0.1",
|
||||
@@ -142,42 +142,42 @@
|
||||
"remark-parse": "^11.0.0",
|
||||
"remark-rehype": "^11.1.2",
|
||||
"rrweb-player": "2.0.0-alpha.20",
|
||||
"short-unique-id": "^5.0.3",
|
||||
"slugify": "^1.6.6",
|
||||
"sonner": "^1.4.0",
|
||||
"short-unique-id": "^5.3.2",
|
||||
"slugify": "^1.6.8",
|
||||
"sonner": "^1.7.4",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^2.2.2",
|
||||
"tailwind-merge": "^3.0.2",
|
||||
"tailwindcss": "^4.0.6",
|
||||
"superjson": "^2.2.6",
|
||||
"tailwind-merge": "^3.5.0",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"tw-animate-css": "^1.3.6",
|
||||
"usehooks-ts": "^2.14.0",
|
||||
"vite-tsconfig-paths": "^5.1.4",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
"usehooks-ts": "^2.16.0",
|
||||
"vite-tsconfig-paths": "^6.1.1",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "1.9.4",
|
||||
"@cloudflare/vite-plugin": "1.20.3",
|
||||
"@biomejs/biome": "2.4.10",
|
||||
"@cloudflare/vite-plugin": "1.30.2",
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/trpc": "workspace:*",
|
||||
"@tanstack/devtools-event-client": "^0.3.3",
|
||||
"@testing-library/dom": "^10.4.0",
|
||||
"@testing-library/react": "^16.2.0",
|
||||
"@tanstack/devtools-event-client": "^0.4.3",
|
||||
"@testing-library/dom": "^10.4.1",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@types/lodash.debounce": "^4.0.9",
|
||||
"@types/lodash.isequal": "^4.5.8",
|
||||
"@types/lodash.throttle": "^4.1.9",
|
||||
"@types/ramda": "^0.31.0",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/react": "catalog:",
|
||||
"@types/react-dom": "catalog:",
|
||||
"@types/react-grid-layout": "^1.3.5",
|
||||
"@types/react-simple-maps": "^3.0.4",
|
||||
"@types/react-syntax-highlighter": "^15.5.11",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"jsdom": "^26.0.0",
|
||||
"@types/react-grid-layout": "^2.1.0",
|
||||
"@types/react-simple-maps": "^3.0.6",
|
||||
"@types/react-syntax-highlighter": "^15.5.13",
|
||||
"@vitejs/plugin-react": "^4.7.0",
|
||||
"jsdom": "^26.1.0",
|
||||
"typescript": "catalog:",
|
||||
"vite": "^6.3.5",
|
||||
"vitest": "^3.0.5",
|
||||
"web-vitals": "^4.2.4",
|
||||
"wrangler": "4.59.1"
|
||||
"vite": "^6.4.1",
|
||||
"vitest": "^3.2.4",
|
||||
"web-vitals": "^5.2.0",
|
||||
"wrangler": "4.78.0"
|
||||
}
|
||||
}
|
||||
@@ -1,61 +1,25 @@
|
||||
import { TooltipComplete } from '@/components/tooltip-complete';
|
||||
import { useDebounceState } from '@/hooks/use-debounce-state';
|
||||
import useWS from '@/hooks/use-ws';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import { cn } from '@/utils/cn';
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useQueryClient } from '@tanstack/react-query';
|
||||
import { useCallback } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { AnimatedNumber } from '../animated-number';
|
||||
import { TooltipComplete } from '@/components/tooltip-complete';
|
||||
import { useLiveCounter } from '@/hooks/use-live-counter';
|
||||
import { cn } from '@/utils/cn';
|
||||
|
||||
export interface LiveCounterProps {
|
||||
projectId: string;
|
||||
shareId?: string;
|
||||
}
|
||||
|
||||
const FIFTEEN_SECONDS = 1000 * 30;
|
||||
|
||||
export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
|
||||
const trpc = useTRPC();
|
||||
const client = useQueryClient();
|
||||
const counter = useDebounceState(0, 1000);
|
||||
const lastRefresh = useRef(Date.now());
|
||||
const query = useQuery(
|
||||
trpc.overview.liveVisitors.queryOptions({
|
||||
projectId,
|
||||
shareId,
|
||||
}),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (query.data) {
|
||||
counter.set(query.data);
|
||||
}
|
||||
}, [query.data]);
|
||||
|
||||
useWS<number>(
|
||||
`/live/visitors/${projectId}`,
|
||||
(value) => {
|
||||
if (!Number.isNaN(value)) {
|
||||
counter.set(value);
|
||||
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
|
||||
lastRefresh.current = Date.now();
|
||||
if (!document.hidden) {
|
||||
toast('Refreshed data');
|
||||
client.refetchQueries({
|
||||
type: 'active',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
debounce: {
|
||||
delay: 1000,
|
||||
maxWait: 5000,
|
||||
},
|
||||
},
|
||||
);
|
||||
const onRefresh = useCallback(() => {
|
||||
toast('Refreshed data');
|
||||
client.refetchQueries({
|
||||
type: 'active',
|
||||
});
|
||||
}, [client]);
|
||||
const counter = useLiveCounter({ projectId, shareId, onRefresh });
|
||||
|
||||
return (
|
||||
<TooltipComplete
|
||||
@@ -66,13 +30,13 @@ export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
|
||||
<div
|
||||
className={cn(
|
||||
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
|
||||
counter.debounced === 0 && 'bg-destructive opacity-0',
|
||||
counter.debounced === 0 && 'bg-destructive opacity-0'
|
||||
)}
|
||||
/>
|
||||
<div
|
||||
className={cn(
|
||||
'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
|
||||
counter.debounced === 0 && 'bg-destructive',
|
||||
'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
|
||||
counter.debounced === 0 && 'bg-destructive'
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
LayoutDashboardIcon,
|
||||
LayoutPanelTopIcon,
|
||||
PlusIcon,
|
||||
ScrollTextIcon,
|
||||
SearchIcon,
|
||||
SparklesIcon,
|
||||
TrendingUpDownIcon,
|
||||
@@ -61,6 +62,7 @@ export default function SidebarProjectMenu({
|
||||
<SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" />
|
||||
<SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" />
|
||||
<SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" />
|
||||
<SidebarLink href={'/logs'} icon={ScrollTextIcon} label="Logs" />
|
||||
<SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" />
|
||||
<SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" />
|
||||
<SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" />
|
||||
|
||||
81
apps/start/src/hooks/use-live-counter.ts
Normal file
81
apps/start/src/hooks/use-live-counter.ts
Normal file
@@ -0,0 +1,81 @@
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useDebounceState } from './use-debounce-state';
|
||||
import useWS from './use-ws';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
|
||||
const FIFTEEN_SECONDS = 1000 * 15;
|
||||
/** Refetch from API when WS-only updates may be stale (e.g. visitors left). */
|
||||
const FALLBACK_STALE_MS = 1000 * 60;
|
||||
|
||||
export function useLiveCounter({
|
||||
projectId,
|
||||
shareId,
|
||||
onRefresh,
|
||||
}: {
|
||||
projectId: string;
|
||||
shareId?: string;
|
||||
onRefresh?: () => void;
|
||||
}) {
|
||||
const trpc = useTRPC();
|
||||
const queryClient = useQueryClient();
|
||||
const counter = useDebounceState(0, 1000);
|
||||
const lastRefresh = useRef(Date.now());
|
||||
const query = useQuery(
|
||||
trpc.overview.liveVisitors.queryOptions({
|
||||
projectId,
|
||||
shareId: shareId ?? undefined,
|
||||
})
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (query.data) {
|
||||
counter.set(query.data);
|
||||
}
|
||||
}, [query.data]);
|
||||
|
||||
useWS<number>(
|
||||
`/live/visitors/${projectId}`,
|
||||
(value) => {
|
||||
if (!Number.isNaN(value)) {
|
||||
counter.set(value);
|
||||
if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
|
||||
lastRefresh.current = Date.now();
|
||||
if (!document.hidden) {
|
||||
onRefresh?.();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
debounce: {
|
||||
delay: 1000,
|
||||
maxWait: 5000,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const id = setInterval(async () => {
|
||||
if (Date.now() - lastRefresh.current < FALLBACK_STALE_MS) {
|
||||
return;
|
||||
}
|
||||
const data = await queryClient.fetchQuery(
|
||||
trpc.overview.liveVisitors.queryOptions(
|
||||
{
|
||||
projectId,
|
||||
shareId: shareId ?? undefined,
|
||||
},
|
||||
// Default query staleTime is 5m; bypass cache so this reconciliation always hits the API.
|
||||
{ staleTime: 0 }
|
||||
)
|
||||
);
|
||||
counter.set(data);
|
||||
lastRefresh.current = Date.now();
|
||||
}, FALLBACK_STALE_MS);
|
||||
|
||||
return () => clearInterval(id);
|
||||
}, [projectId, shareId, trpc, queryClient, counter.set]);
|
||||
|
||||
return counter;
|
||||
}
|
||||
@@ -47,6 +47,7 @@ import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/
|
||||
import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references'
|
||||
import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime'
|
||||
import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages'
|
||||
import { Route as AppOrganizationIdProjectIdLogsRouteImport } from './routes/_app.$organizationId.$projectId.logs'
|
||||
import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights'
|
||||
import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups'
|
||||
import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards'
|
||||
@@ -352,6 +353,12 @@ const AppOrganizationIdProjectIdPagesRoute =
|
||||
path: '/pages',
|
||||
getParentRoute: () => AppOrganizationIdProjectIdRoute,
|
||||
} as any)
|
||||
const AppOrganizationIdProjectIdLogsRoute =
|
||||
AppOrganizationIdProjectIdLogsRouteImport.update({
|
||||
id: '/logs',
|
||||
path: '/logs',
|
||||
getParentRoute: () => AppOrganizationIdProjectIdRoute,
|
||||
} as any)
|
||||
const AppOrganizationIdProjectIdInsightsRoute =
|
||||
AppOrganizationIdProjectIdInsightsRouteImport.update({
|
||||
id: '/insights',
|
||||
@@ -660,6 +667,7 @@ export interface FileRoutesByFullPath {
|
||||
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -738,6 +746,7 @@ export interface FileRoutesByTo {
|
||||
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -814,6 +823,7 @@ export interface FileRoutesById {
|
||||
'/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/_app/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -905,6 +915,7 @@ export interface FileRouteTypes {
|
||||
| '/$organizationId/$projectId/dashboards'
|
||||
| '/$organizationId/$projectId/groups'
|
||||
| '/$organizationId/$projectId/insights'
|
||||
| '/$organizationId/$projectId/logs'
|
||||
| '/$organizationId/$projectId/pages'
|
||||
| '/$organizationId/$projectId/realtime'
|
||||
| '/$organizationId/$projectId/references'
|
||||
@@ -983,6 +994,7 @@ export interface FileRouteTypes {
|
||||
| '/$organizationId/$projectId/dashboards'
|
||||
| '/$organizationId/$projectId/groups'
|
||||
| '/$organizationId/$projectId/insights'
|
||||
| '/$organizationId/$projectId/logs'
|
||||
| '/$organizationId/$projectId/pages'
|
||||
| '/$organizationId/$projectId/realtime'
|
||||
| '/$organizationId/$projectId/references'
|
||||
@@ -1058,6 +1070,7 @@ export interface FileRouteTypes {
|
||||
| '/_app/$organizationId/$projectId/dashboards'
|
||||
| '/_app/$organizationId/$projectId/groups'
|
||||
| '/_app/$organizationId/$projectId/insights'
|
||||
| '/_app/$organizationId/$projectId/logs'
|
||||
| '/_app/$organizationId/$projectId/pages'
|
||||
| '/_app/$organizationId/$projectId/realtime'
|
||||
| '/_app/$organizationId/$projectId/references'
|
||||
@@ -1444,6 +1457,13 @@ declare module '@tanstack/react-router' {
|
||||
preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport
|
||||
parentRoute: typeof AppOrganizationIdProjectIdRoute
|
||||
}
|
||||
'/_app/$organizationId/$projectId/logs': {
|
||||
id: '/_app/$organizationId/$projectId/logs'
|
||||
path: '/logs'
|
||||
fullPath: '/$organizationId/$projectId/logs'
|
||||
preLoaderRoute: typeof AppOrganizationIdProjectIdLogsRouteImport
|
||||
parentRoute: typeof AppOrganizationIdProjectIdRoute
|
||||
}
|
||||
'/_app/$organizationId/$projectId/insights': {
|
||||
id: '/_app/$organizationId/$projectId/insights'
|
||||
path: '/insights'
|
||||
@@ -2028,6 +2048,7 @@ interface AppOrganizationIdProjectIdRouteChildren {
|
||||
AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
AppOrganizationIdProjectIdLogsRoute: typeof AppOrganizationIdProjectIdLogsRoute
|
||||
AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute
|
||||
AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -2054,6 +2075,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh
|
||||
AppOrganizationIdProjectIdGroupsRoute,
|
||||
AppOrganizationIdProjectIdInsightsRoute:
|
||||
AppOrganizationIdProjectIdInsightsRoute,
|
||||
AppOrganizationIdProjectIdLogsRoute: AppOrganizationIdProjectIdLogsRoute,
|
||||
AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute,
|
||||
AppOrganizationIdProjectIdRealtimeRoute:
|
||||
AppOrganizationIdProjectIdRealtimeRoute,
|
||||
|
||||
340
apps/start/src/routes/_app.$organizationId.$projectId.logs.tsx
Normal file
340
apps/start/src/routes/_app.$organizationId.$projectId.logs.tsx
Normal file
@@ -0,0 +1,340 @@
|
||||
import { useInfiniteQuery, useQuery } from '@tanstack/react-query';
|
||||
import { createFileRoute } from '@tanstack/react-router';
|
||||
import {
|
||||
AlertCircleIcon,
|
||||
AlertTriangleIcon,
|
||||
BugIcon,
|
||||
ChevronDownIcon,
|
||||
ChevronRightIcon,
|
||||
InfoIcon,
|
||||
SearchIcon,
|
||||
XCircleIcon,
|
||||
} from 'lucide-react';
|
||||
import { parseAsArrayOf, parseAsString, useQueryState } from 'nuqs';
|
||||
import { useState } from 'react';
|
||||
import { PageContainer } from '@/components/page-container';
|
||||
import { PageHeader } from '@/components/page-header';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import { cn } from '@/utils/cn';
|
||||
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
|
||||
import type { IServiceLog } from '@openpanel/trpc';
|
||||
|
||||
export const Route = createFileRoute(
|
||||
'/_app/$organizationId/$projectId/logs',
|
||||
)({
|
||||
component: Component,
|
||||
head: () => ({
|
||||
meta: [{ title: createProjectTitle(PAGE_TITLES.LOGS) }],
|
||||
}),
|
||||
});
|
||||
|
||||
const SEVERITY_LEVELS = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'] as const;
|
||||
|
||||
type SeverityLevel = (typeof SEVERITY_LEVELS)[number];
|
||||
|
||||
function getSeverityVariant(
|
||||
severity: string,
|
||||
): 'default' | 'secondary' | 'info' | 'warning' | 'destructive' | 'outline' {
|
||||
switch (severity.toLowerCase()) {
|
||||
case 'fatal':
|
||||
case 'critical':
|
||||
case 'error':
|
||||
return 'destructive';
|
||||
case 'warn':
|
||||
case 'warning':
|
||||
return 'warning';
|
||||
case 'info':
|
||||
return 'info';
|
||||
default:
|
||||
return 'outline';
|
||||
}
|
||||
}
|
||||
|
||||
function SeverityIcon({ severity }: { severity: string }) {
|
||||
const s = severity.toLowerCase();
|
||||
const cls = 'size-3.5 shrink-0';
|
||||
if (s === 'fatal' || s === 'critical') return <XCircleIcon className={cn(cls, 'text-destructive')} />;
|
||||
if (s === 'error') return <AlertCircleIcon className={cn(cls, 'text-destructive')} />;
|
||||
if (s === 'warn' || s === 'warning') return <AlertTriangleIcon className={cn(cls, 'text-yellow-500')} />;
|
||||
if (s === 'debug' || s === 'trace') return <BugIcon className={cn(cls, 'text-muted-foreground')} />;
|
||||
return <InfoIcon className={cn(cls, 'text-blue-500')} />;
|
||||
}
|
||||
|
||||
function LogRow({ log }: { log: IServiceLog }) {
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const hasDetails =
|
||||
(log.attributes && Object.keys(log.attributes).length > 0) ||
|
||||
(log.resource && Object.keys(log.resource).length > 0) ||
|
||||
log.traceId ||
|
||||
log.loggerName;
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(
|
||||
'grid w-full grid-cols-[28px_90px_80px_1fr_120px] items-center gap-3 px-4 py-2.5 text-left text-sm transition-colors hover:bg-muted/50 border-b border-border/50',
|
||||
expanded && 'bg-muted/30',
|
||||
)}
|
||||
onClick={() => hasDetails && setExpanded((v) => !v)}
|
||||
>
|
||||
<span className="flex items-center justify-center text-muted-foreground">
|
||||
{hasDetails ? (
|
||||
expanded ? (
|
||||
<ChevronDownIcon className="size-3.5" />
|
||||
) : (
|
||||
<ChevronRightIcon className="size-3.5" />
|
||||
)
|
||||
) : null}
|
||||
</span>
|
||||
<span className="text-muted-foreground tabular-nums text-xs">
|
||||
{log.timestamp.toLocaleTimeString()}
|
||||
</span>
|
||||
<span>
|
||||
<Badge variant={getSeverityVariant(log.severityText)} className="gap-1 font-mono uppercase text-[10px]">
|
||||
<SeverityIcon severity={log.severityText} />
|
||||
{log.severityText}
|
||||
</Badge>
|
||||
</span>
|
||||
<span className="truncate font-mono text-xs">{log.body}</span>
|
||||
<span className="truncate text-xs text-muted-foreground text-right">
|
||||
{log.loggerName || log.os || log.device || '—'}
|
||||
</span>
|
||||
</button>
|
||||
|
||||
{expanded && (
|
||||
<div className="border-b border-border/50 bg-muted/20 px-4 py-3">
|
||||
<div className="grid gap-4 pl-[calc(28px+0.75rem)] text-xs sm:grid-cols-2">
|
||||
{log.loggerName && (
|
||||
<MetaRow label="Logger" value={log.loggerName} />
|
||||
)}
|
||||
{log.traceId && (
|
||||
<MetaRow label="Trace ID" value={log.traceId} mono />
|
||||
)}
|
||||
{log.spanId && (
|
||||
<MetaRow label="Span ID" value={log.spanId} mono />
|
||||
)}
|
||||
{log.deviceId && (
|
||||
<MetaRow label="Device ID" value={log.deviceId} mono />
|
||||
)}
|
||||
{log.os && (
|
||||
<MetaRow label="OS" value={`${log.os} ${log.osVersion}`.trim()} />
|
||||
)}
|
||||
{log.country && (
|
||||
<MetaRow label="Location" value={[log.city, log.country].filter(Boolean).join(', ')} />
|
||||
)}
|
||||
|
||||
{Object.keys(log.attributes).length > 0 && (
|
||||
<div className="col-span-2">
|
||||
<p className="mb-1.5 font-semibold text-muted-foreground">Attributes</p>
|
||||
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
|
||||
{Object.entries(log.attributes).map(([k, v]) => (
|
||||
<div key={k} className="flex gap-2">
|
||||
<span className="text-blue-500 shrink-0">{k}</span>
|
||||
<span className="text-muted-foreground">=</span>
|
||||
<span className="break-all">{v}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{Object.keys(log.resource).length > 0 && (
|
||||
<div className="col-span-2">
|
||||
<p className="mb-1.5 font-semibold text-muted-foreground">Resource</p>
|
||||
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
|
||||
{Object.entries(log.resource).map(([k, v]) => (
|
||||
<div key={k} className="flex gap-2">
|
||||
<span className="text-emerald-600 shrink-0">{k}</span>
|
||||
<span className="text-muted-foreground">=</span>
|
||||
<span className="break-all">{v}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function MetaRow({ label, value, mono }: { label: string; value: string; mono?: boolean }) {
|
||||
return (
|
||||
<div className="flex gap-2">
|
||||
<span className="w-24 shrink-0 font-medium text-muted-foreground">{label}</span>
|
||||
<span className={cn('break-all', mono && 'font-mono')}>{value}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Component() {
|
||||
const { projectId } = Route.useParams();
|
||||
const trpc = useTRPC();
|
||||
|
||||
const [search, setSearch] = useQueryState('search', parseAsString.withDefault(''));
|
||||
const [severities, setSeverities] = useQueryState(
|
||||
'severity',
|
||||
parseAsArrayOf(parseAsString).withDefault([]),
|
||||
);
|
||||
|
||||
const countsQuery = useQuery(
|
||||
trpc.log.severityCounts.queryOptions({ projectId }),
|
||||
);
|
||||
|
||||
const logsQuery = useInfiniteQuery(
|
||||
trpc.log.list.infiniteQueryOptions(
|
||||
{
|
||||
projectId,
|
||||
search: search || undefined,
|
||||
severity: severities.length > 0 ? (severities as SeverityLevel[]) : undefined,
|
||||
take: 50,
|
||||
},
|
||||
{
|
||||
getNextPageParam: (lastPage) => lastPage.meta.next,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const logs = logsQuery.data?.pages.flatMap((p) => p.data) ?? [];
|
||||
const counts = countsQuery.data ?? {};
|
||||
|
||||
const toggleSeverity = (s: string) => {
|
||||
setSeverities((prev) =>
|
||||
prev.includes(s) ? prev.filter((x) => x !== s) : [...prev, s],
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<PageContainer>
|
||||
<PageHeader
|
||||
className="mb-6"
|
||||
title="Logs"
|
||||
description="Captured device and application logs in OpenTelemetry format"
|
||||
/>
|
||||
|
||||
{/* Severity filter chips */}
|
||||
<div className="mb-4 flex flex-wrap items-center gap-2">
|
||||
{SEVERITY_LEVELS.map((s) => {
|
||||
const active = severities.includes(s);
|
||||
const count = counts[s] ?? 0;
|
||||
return (
|
||||
<button
|
||||
key={s}
|
||||
type="button"
|
||||
onClick={() => toggleSeverity(s)}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1.5 rounded-full border px-3 py-1 text-xs font-medium transition-colors',
|
||||
active
|
||||
? 'border-foreground bg-foreground text-background'
|
||||
: 'border-border bg-transparent text-muted-foreground hover:border-foreground/50 hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
<SeverityIcon severity={s} />
|
||||
<span className="uppercase">{s}</span>
|
||||
{count > 0 && (
|
||||
<span className={cn('tabular-nums', active ? 'opacity-70' : 'opacity-50')}>
|
||||
{count.toLocaleString()}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
|
||||
{severities.length > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSeverities([])}
|
||||
className="text-xs text-muted-foreground hover:text-foreground underline"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Search */}
|
||||
<div className="mb-4 relative">
|
||||
<SearchIcon className="absolute left-3 top-1/2 -translate-y-1/2 size-4 text-muted-foreground" />
|
||||
<Input
|
||||
className="pl-9"
|
||||
placeholder="Search log messages…"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value || null)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Log table */}
|
||||
<div className="rounded-lg border bg-card overflow-hidden">
|
||||
{/* Header */}
|
||||
<div className="grid grid-cols-[28px_90px_80px_1fr_120px] gap-3 border-b bg-muted/50 px-4 py-2 text-xs font-medium text-muted-foreground">
|
||||
<span />
|
||||
<span>Time</span>
|
||||
<span>Level</span>
|
||||
<span>Message</span>
|
||||
<span className="text-right">Source</span>
|
||||
</div>
|
||||
|
||||
{logsQuery.isPending && (
|
||||
<div className="space-y-0 divide-y">
|
||||
{Array.from({ length: 10 }).map((_, i) => (
|
||||
<div key={i} className="h-10 animate-pulse bg-muted/30" />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!logsQuery.isPending && logs.length === 0 && (
|
||||
<div className="flex flex-col items-center gap-2 py-16 text-center text-muted-foreground">
|
||||
<ScrollTextIcon className="size-10 opacity-30" />
|
||||
<p className="text-sm font-medium">No logs found</p>
|
||||
<p className="text-xs">
|
||||
{search || severities.length > 0
|
||||
? 'Try adjusting your filters'
|
||||
: 'Logs will appear here once your app starts sending them'}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{logs.map((log) => (
|
||||
<LogRow key={log.id} log={log} />
|
||||
))}
|
||||
|
||||
{logsQuery.hasNextPage && (
|
||||
<div className="flex justify-center p-4">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => logsQuery.fetchNextPage()}
|
||||
disabled={logsQuery.isFetchingNextPage}
|
||||
>
|
||||
{logsQuery.isFetchingNextPage ? 'Loading…' : 'Load more'}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</PageContainer>
|
||||
);
|
||||
}
|
||||
|
||||
function ScrollTextIcon({ className }: { className?: string }) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className={className}
|
||||
>
|
||||
<path d="M8 21h12a2 2 0 0 0 2-2v-2H10v2a2 2 0 1 1-4 0V5a2 2 0 1 0-4 0v3h4" />
|
||||
<path d="M19 17V5a2 2 0 0 0-2-2H4" />
|
||||
<path d="M15 8h-5" />
|
||||
<path d="M15 12h-5" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
import { AnimatedNumber } from '@/components/animated-number';
|
||||
import { Ping } from '@/components/ping';
|
||||
import { useNumber } from '@/hooks/use-numer-formatter';
|
||||
import useWS from '@/hooks/use-ws';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import type { RouterOutputs } from '@/trpc/client';
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { createFileRoute } from '@tanstack/react-router';
|
||||
import { z } from 'zod';
|
||||
import { AnimatedNumber } from '@/components/animated-number';
|
||||
import { Ping } from '@/components/ping';
|
||||
import useWS from '@/hooks/use-ws';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import type { RouterOutputs } from '@/trpc/client';
|
||||
|
||||
const widgetSearchSchema = z.object({
|
||||
shareId: z.string(),
|
||||
@@ -20,33 +19,33 @@ export const Route = createFileRoute('/widget/counter')({
|
||||
});
|
||||
|
||||
function RouteComponent() {
|
||||
const { shareId, limit, color } = Route.useSearch();
|
||||
const { shareId } = Route.useSearch();
|
||||
const trpc = useTRPC();
|
||||
|
||||
// Fetch widget data
|
||||
const { data, isLoading } = useQuery(
|
||||
trpc.widget.counter.queryOptions({ shareId }),
|
||||
trpc.widget.counter.queryOptions({ shareId })
|
||||
);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 px-2 h-8">
|
||||
<div className="flex h-8 items-center gap-2 px-2">
|
||||
<Ping />
|
||||
<AnimatedNumber value={0} suffix=" unique visitors" />
|
||||
<AnimatedNumber suffix=" unique visitors" value={0} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 px-2 h-8">
|
||||
<div className="flex h-8 items-center gap-2 px-2">
|
||||
<Ping className="bg-orange-500" />
|
||||
<AnimatedNumber value={0} suffix=" unique visitors" />
|
||||
<AnimatedNumber suffix=" unique visitors" value={0} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return <CounterWidget shareId={shareId} data={data} />;
|
||||
return <CounterWidget data={data} shareId={shareId} />;
|
||||
}
|
||||
|
||||
interface RealtimeWidgetProps {
|
||||
@@ -57,30 +56,29 @@ interface RealtimeWidgetProps {
|
||||
function CounterWidget({ shareId, data }: RealtimeWidgetProps) {
|
||||
const trpc = useTRPC();
|
||||
const queryClient = useQueryClient();
|
||||
const number = useNumber();
|
||||
|
||||
// WebSocket subscription for real-time updates
|
||||
useWS<number>(
|
||||
`/live/visitors/${data.projectId}`,
|
||||
(res) => {
|
||||
() => {
|
||||
if (!document.hidden) {
|
||||
queryClient.refetchQueries(
|
||||
trpc.widget.counter.queryFilter({ shareId }),
|
||||
trpc.widget.counter.queryFilter({ shareId })
|
||||
);
|
||||
}
|
||||
},
|
||||
{
|
||||
debounce: {
|
||||
delay: 1000,
|
||||
maxWait: 60000,
|
||||
maxWait: 60_000,
|
||||
},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2 px-2 h-8">
|
||||
<div className="flex h-8 items-center gap-2 px-2">
|
||||
<Ping />
|
||||
<AnimatedNumber value={data.counter} suffix=" unique visitors" />
|
||||
<AnimatedNumber suffix=" unique visitors" value={data.counter} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,15 @@
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { createFileRoute } from '@tanstack/react-router';
|
||||
import type React from 'react';
|
||||
import {
|
||||
Bar,
|
||||
BarChart,
|
||||
ResponsiveContainer,
|
||||
Tooltip,
|
||||
XAxis,
|
||||
YAxis,
|
||||
} from 'recharts';
|
||||
import { z } from 'zod';
|
||||
import { AnimatedNumber } from '@/components/animated-number';
|
||||
import {
|
||||
ChartTooltipContainer,
|
||||
@@ -14,18 +26,6 @@ import { countries } from '@/translations/countries';
|
||||
import type { RouterOutputs } from '@/trpc/client';
|
||||
import { cn } from '@/utils/cn';
|
||||
import { getChartColor } from '@/utils/theme';
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { createFileRoute } from '@tanstack/react-router';
|
||||
import type React from 'react';
|
||||
import {
|
||||
Bar,
|
||||
BarChart,
|
||||
ResponsiveContainer,
|
||||
Tooltip,
|
||||
XAxis,
|
||||
YAxis,
|
||||
} from 'recharts';
|
||||
import { z } from 'zod';
|
||||
|
||||
const widgetSearchSchema = z.object({
|
||||
shareId: z.string(),
|
||||
@@ -44,7 +44,7 @@ function RouteComponent() {
|
||||
|
||||
// Fetch widget data
|
||||
const { data: widgetData, isLoading } = useQuery(
|
||||
trpc.widget.realtimeData.queryOptions({ shareId }),
|
||||
trpc.widget.realtimeData.queryOptions({ shareId })
|
||||
);
|
||||
|
||||
if (isLoading) {
|
||||
@@ -53,10 +53,10 @@ function RouteComponent() {
|
||||
|
||||
if (!widgetData) {
|
||||
return (
|
||||
<div className="flex h-screen w-full center-center bg-background text-foreground col p-4">
|
||||
<LogoSquare className="size-10 mb-4" />
|
||||
<h1 className="text-xl font-semibold">Widget not found</h1>
|
||||
<p className="mt-2 text-sm text-muted-foreground">
|
||||
<div className="center-center col flex h-screen w-full bg-background p-4 text-foreground">
|
||||
<LogoSquare className="mb-4 size-10" />
|
||||
<h1 className="font-semibold text-xl">Widget not found</h1>
|
||||
<p className="mt-2 text-muted-foreground text-sm">
|
||||
This widget is not available or has been removed.
|
||||
</p>
|
||||
</div>
|
||||
@@ -65,10 +65,10 @@ function RouteComponent() {
|
||||
|
||||
return (
|
||||
<RealtimeWidget
|
||||
shareId={shareId}
|
||||
limit={limit}
|
||||
data={widgetData}
|
||||
color={color}
|
||||
data={widgetData}
|
||||
limit={limit}
|
||||
shareId={shareId}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -83,7 +83,6 @@ interface RealtimeWidgetProps {
|
||||
function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
const trpc = useTRPC();
|
||||
const queryClient = useQueryClient();
|
||||
const number = useNumber();
|
||||
|
||||
// WebSocket subscription for real-time updates
|
||||
useWS<number>(
|
||||
@@ -91,16 +90,16 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
() => {
|
||||
if (!document.hidden) {
|
||||
queryClient.refetchQueries(
|
||||
trpc.widget.realtimeData.queryFilter({ shareId }),
|
||||
trpc.widget.realtimeData.queryFilter({ shareId })
|
||||
);
|
||||
}
|
||||
},
|
||||
{
|
||||
debounce: {
|
||||
delay: 1000,
|
||||
maxWait: 60000,
|
||||
maxWait: 60_000,
|
||||
},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
const maxDomain =
|
||||
@@ -111,8 +110,12 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
const referrers = data.referrers.length > 0 ? 1 : 0;
|
||||
const paths = data.paths.length > 0 ? 1 : 0;
|
||||
const value = countries + referrers + paths;
|
||||
if (value === 3) return 'md:grid-cols-3';
|
||||
if (value === 2) return 'md:grid-cols-2';
|
||||
if (value === 3) {
|
||||
return 'md:grid-cols-3';
|
||||
}
|
||||
if (value === 2) {
|
||||
return 'md:grid-cols-2';
|
||||
}
|
||||
return 'md:grid-cols-1';
|
||||
})();
|
||||
|
||||
@@ -120,10 +123,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
<div className="flex h-screen w-full flex-col bg-background text-foreground">
|
||||
{/* Header with live counter */}
|
||||
<div className="p-6 pb-3">
|
||||
<div className="flex items-center justify-between w-full h-4">
|
||||
<div className="flex items-center gap-3 w-full">
|
||||
<div className="flex h-4 w-full items-center justify-between">
|
||||
<div className="flex w-full items-center gap-3">
|
||||
<Ping />
|
||||
<div className="text-sm font-medium text-muted-foreground flex-1">
|
||||
<div className="flex-1 font-medium text-muted-foreground text-sm">
|
||||
USERS IN LAST 30 MINUTES
|
||||
</div>
|
||||
{data.project.domain && <SerieIcon name={data.project.domain} />}
|
||||
@@ -131,14 +134,14 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
</div>
|
||||
|
||||
<div className="row">
|
||||
<div className="font-mono text-6xl font-bold h-18 text-foreground">
|
||||
<div className="h-18 font-bold font-mono text-6xl text-foreground">
|
||||
<AnimatedNumber value={data.liveCount} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex h-20 w-full flex-col -mt-4">
|
||||
<div className="-mt-4 flex h-20 w-full flex-col">
|
||||
<div className="flex-1">
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
<ResponsiveContainer height="100%" width="100%">
|
||||
<BarChart
|
||||
data={data.histogram}
|
||||
margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
|
||||
@@ -148,22 +151,22 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
cursor={{ fill: 'var(--def-100)', radius: 4 }}
|
||||
/>
|
||||
<XAxis
|
||||
dataKey="time"
|
||||
axisLine={false}
|
||||
tickLine={false}
|
||||
dataKey="time"
|
||||
interval="preserveStartEnd"
|
||||
tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }}
|
||||
tickLine={false}
|
||||
ticks={[
|
||||
data.histogram[0].time,
|
||||
data.histogram[data.histogram.length - 1].time,
|
||||
]}
|
||||
interval="preserveStartEnd"
|
||||
/>
|
||||
<YAxis hide domain={[0, maxDomain]} />
|
||||
<YAxis domain={[0, maxDomain]} hide />
|
||||
<Bar
|
||||
dataKey="sessionCount"
|
||||
fill={color || 'var(--chart-0)'}
|
||||
isAnimationActive={false}
|
||||
radius={[4, 4, 4, 4]}
|
||||
fill={color || 'var(--chart-0)'}
|
||||
/>
|
||||
</BarChart>
|
||||
</ResponsiveContainer>
|
||||
@@ -174,24 +177,24 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
{(data.countries.length > 0 ||
|
||||
data.referrers.length > 0 ||
|
||||
data.paths.length > 0) && (
|
||||
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar border-t">
|
||||
<div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto border-t p-6">
|
||||
<div className={cn('grid grid-cols-1 gap-6', grids)}>
|
||||
{/* Countries */}
|
||||
{data.countries.length > 0 && (
|
||||
<div className="flex flex-col">
|
||||
<div className="mb-3 text-xs font-medium text-muted-foreground">
|
||||
<div className="mb-3 font-medium text-muted-foreground text-xs">
|
||||
COUNTRY
|
||||
</div>
|
||||
<div className="col">
|
||||
{(() => {
|
||||
const { visible, rest, restCount } = getRestItems(
|
||||
data.countries,
|
||||
limit,
|
||||
limit
|
||||
);
|
||||
return (
|
||||
<>
|
||||
{visible.map((item) => (
|
||||
<RowItem key={item.country} count={item.count}>
|
||||
<RowItem count={item.count} key={item.country}>
|
||||
<div className="flex items-center gap-2">
|
||||
<SerieIcon name={item.country} />
|
||||
<span className="text-sm">
|
||||
@@ -224,19 +227,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
{/* Referrers */}
|
||||
{data.referrers.length > 0 && (
|
||||
<div className="flex flex-col">
|
||||
<div className="mb-3 text-xs font-medium text-muted-foreground">
|
||||
<div className="mb-3 font-medium text-muted-foreground text-xs">
|
||||
REFERRER
|
||||
</div>
|
||||
<div className="col">
|
||||
{(() => {
|
||||
const { visible, rest, restCount } = getRestItems(
|
||||
data.referrers,
|
||||
limit,
|
||||
limit
|
||||
);
|
||||
return (
|
||||
<>
|
||||
{visible.map((item) => (
|
||||
<RowItem key={item.referrer} count={item.count}>
|
||||
<RowItem count={item.count} key={item.referrer}>
|
||||
<div className="flex items-center gap-2">
|
||||
<SerieIcon name={item.referrer} />
|
||||
<span className="truncate text-sm">
|
||||
@@ -263,19 +266,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
{/* Paths */}
|
||||
{data.paths.length > 0 && (
|
||||
<div className="flex flex-col">
|
||||
<div className="mb-3 text-xs font-medium text-muted-foreground">
|
||||
<div className="mb-3 font-medium text-muted-foreground text-xs">
|
||||
PATH
|
||||
</div>
|
||||
<div className="col">
|
||||
{(() => {
|
||||
const { visible, rest, restCount } = getRestItems(
|
||||
data.paths,
|
||||
limit,
|
||||
limit
|
||||
);
|
||||
return (
|
||||
<>
|
||||
{visible.map((item) => (
|
||||
<RowItem key={item.path} count={item.count}>
|
||||
<RowItem count={item.count} key={item.path}>
|
||||
<span className="truncate text-sm">
|
||||
{item.path}
|
||||
</span>
|
||||
@@ -303,10 +306,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
|
||||
}
|
||||
|
||||
// Custom tooltip component that uses portals to escape overflow hidden
|
||||
const CustomTooltip = ({ active, payload, coordinate }: any) => {
|
||||
const CustomTooltip = ({ active, payload }: any) => {
|
||||
const number = useNumber();
|
||||
|
||||
if (!active || !payload || !payload.length) {
|
||||
if (!(active && payload && payload.length)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -328,10 +331,13 @@ const CustomTooltip = ({ active, payload, coordinate }: any) => {
|
||||
function RowItem({
|
||||
children,
|
||||
count,
|
||||
}: { children: React.ReactNode; count: number }) {
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
count: number;
|
||||
}) {
|
||||
const number = useNumber();
|
||||
return (
|
||||
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
|
||||
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
|
||||
{children}
|
||||
<span className="font-semibold">{number.short(count)}</span>
|
||||
</div>
|
||||
@@ -340,7 +346,7 @@ function RowItem({
|
||||
|
||||
function getRestItems<T extends { count: number }>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
limit: number
|
||||
): { visible: T[]; rest: T[]; restCount: number } {
|
||||
const visible = items.slice(0, limit);
|
||||
const rest = items.slice(limit);
|
||||
@@ -375,7 +381,7 @@ function RestRow({
|
||||
: 'paths';
|
||||
|
||||
return (
|
||||
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
|
||||
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
|
||||
<span className="truncate">
|
||||
{firstName} and {otherCount} more {typeLabel}...
|
||||
</span>
|
||||
@@ -434,13 +440,13 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
|
||||
const itemCount = Math.min(limit, 5);
|
||||
|
||||
return (
|
||||
<div className="flex h-screen w-full flex-col bg-background text-foreground animate-pulse">
|
||||
<div className="flex h-screen w-full animate-pulse flex-col bg-background text-foreground">
|
||||
{/* Header with live counter */}
|
||||
<div className="border-b p-6 pb-3">
|
||||
<div className="flex items-center justify-between w-full h-4">
|
||||
<div className="flex items-center gap-3 w-full">
|
||||
<div className="flex h-4 w-full items-center justify-between">
|
||||
<div className="flex w-full items-center gap-3">
|
||||
<div className="size-2 rounded-full bg-muted" />
|
||||
<div className="text-sm font-medium text-muted-foreground flex-1">
|
||||
<div className="flex-1 font-medium text-muted-foreground text-sm">
|
||||
USERS IN LAST 30 MINUTES
|
||||
</div>
|
||||
</div>
|
||||
@@ -448,35 +454,35 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
|
||||
</div>
|
||||
|
||||
<div className="row">
|
||||
<div className="font-mono text-6xl font-bold h-18 flex items-center py-4 gap-1 row">
|
||||
<div className="h-full w-6 bg-muted rounded" />
|
||||
<div className="h-full w-6 bg-muted rounded" />
|
||||
<div className="row flex h-18 items-center gap-1 py-4 font-bold font-mono text-6xl">
|
||||
<div className="h-full w-6 rounded bg-muted" />
|
||||
<div className="h-full w-6 rounded bg-muted" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex h-20 w-full flex-col -mt-4 pb-2.5">
|
||||
<div className="flex-1 row gap-1 h-full">
|
||||
<div className="-mt-4 flex h-20 w-full flex-col pb-2.5">
|
||||
<div className="row h-full flex-1 gap-1">
|
||||
{SKELETON_HISTOGRAM.map((item, index) => (
|
||||
<div
|
||||
className="mt-auto h-full w-full rounded bg-muted"
|
||||
key={index.toString()}
|
||||
style={{ height: `${item}%` }}
|
||||
className="h-full w-full bg-muted rounded mt-auto"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<div className="row justify-between pt-2">
|
||||
<div className="h-3 w-8 bg-muted rounded" />
|
||||
<div className="h-3 w-8 bg-muted rounded" />
|
||||
<div className="h-3 w-8 rounded bg-muted" />
|
||||
<div className="h-3 w-8 rounded bg-muted" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar">
|
||||
<div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto p-6">
|
||||
{/* Countries, Referrers, and Paths skeleton */}
|
||||
<div className="grid grid-cols-1 gap-6 md:grid-cols-3">
|
||||
{/* Countries skeleton */}
|
||||
<div className="flex flex-col">
|
||||
<div className="mb-3 text-xs font-medium text-muted-foreground">
|
||||
<div className="mb-3 font-medium text-muted-foreground text-xs">
|
||||
COUNTRY
|
||||
</div>
|
||||
<div className="col">
|
||||
@@ -488,7 +494,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
|
||||
|
||||
{/* Referrers skeleton */}
|
||||
<div className="flex flex-col">
|
||||
<div className="mb-3 text-xs font-medium text-muted-foreground">
|
||||
<div className="mb-3 font-medium text-muted-foreground text-xs">
|
||||
REFERRER
|
||||
</div>
|
||||
<div className="col">
|
||||
@@ -500,7 +506,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
|
||||
|
||||
{/* Paths skeleton */}
|
||||
<div className="flex flex-col">
|
||||
<div className="mb-3 text-xs font-medium text-muted-foreground">
|
||||
<div className="mb-3 font-medium text-muted-foreground text-xs">
|
||||
PATH
|
||||
</div>
|
||||
<div className="col">
|
||||
@@ -517,12 +523,12 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
|
||||
|
||||
function RowItemSkeleton() {
|
||||
return (
|
||||
<div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b -mx-3">
|
||||
<div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="size-5 rounded bg-muted" />
|
||||
<div className="h-4 w-24 bg-muted rounded" />
|
||||
<div className="h-4 w-24 rounded bg-muted" />
|
||||
</div>
|
||||
<div className="h-4 w-8 bg-muted rounded" />
|
||||
<div className="h-4 w-8 rounded bg-muted" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ export const PAGE_TITLES = {
|
||||
MEMBERS: 'Members',
|
||||
BILLING: 'Billing',
|
||||
CHAT: 'AI Assistant',
|
||||
LOGS: 'Logs',
|
||||
REALTIME: 'Realtime',
|
||||
REFERENCES: 'References',
|
||||
INSIGHTS: 'Insights',
|
||||
|
||||
@@ -10,15 +10,15 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/web": "workspace:*",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"react-router-dom": "^7.13.1"
|
||||
"react": "^19.2.4",
|
||||
"react-dom": "^19.2.4",
|
||||
"react-router-dom": "^7.13.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^4.7.0",
|
||||
"typescript": "catalog:",
|
||||
"vite": "^6.0.0"
|
||||
"vite": "^6.4.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG NODE_VERSION=22.20.0
|
||||
ARG NODE_VERSION=22.22.2
|
||||
|
||||
FROM node:${NODE_VERSION}-slim AS base
|
||||
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bull-board/api": "6.14.0",
|
||||
"@bull-board/express": "6.14.0",
|
||||
"@bull-board/api": "6.20.6",
|
||||
"@bull-board/express": "6.20.6",
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/email": "workspace:*",
|
||||
@@ -24,24 +24,25 @@
|
||||
"@openpanel/payments": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"bullmq": "^5.63.0",
|
||||
"date-fns": "^3.3.1",
|
||||
"express": "^4.18.2",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"bullmq": "^5.71.1",
|
||||
"date-fns": "^3.6.0",
|
||||
"express": "^4.22.1",
|
||||
"groupmq": "catalog:",
|
||||
"prom-client": "^15.1.3",
|
||||
"ramda": "^0.29.1",
|
||||
"ramda": "^0.32.0",
|
||||
"source-map-support": "^0.5.21",
|
||||
"sqlstring": "^2.3.3",
|
||||
"uuid": "^9.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/express": "^4.17.21",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/source-map-support": "^0.5.10",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"tsdown": "0.14.2",
|
||||
"@types/uuid": "^11.0.0",
|
||||
"tsdown": "0.21.7",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,11 @@ export async function bootCron() {
|
||||
type: 'flushGroups',
|
||||
pattern: 1000 * 10,
|
||||
},
|
||||
{
|
||||
name: 'flush',
|
||||
type: 'flushLogs',
|
||||
pattern: 1000 * 10,
|
||||
},
|
||||
{
|
||||
name: 'insightsDaily',
|
||||
type: 'insightsDaily',
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
gscQueue,
|
||||
importQueue,
|
||||
insightsQueue,
|
||||
logsQueue,
|
||||
miscQueue,
|
||||
notificationQueue,
|
||||
queueLogger,
|
||||
@@ -22,6 +23,7 @@ import { incomingEvent } from './jobs/events.incoming-event';
|
||||
import { gscJob } from './jobs/gsc';
|
||||
import { importJob } from './jobs/import';
|
||||
import { insightsProjectJob } from './jobs/insights';
|
||||
import { incomingLog } from './jobs/logs.incoming-log';
|
||||
import { miscJob } from './jobs/misc';
|
||||
import { notificationJob } from './jobs/notification';
|
||||
import { sessionsJob } from './jobs/sessions';
|
||||
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
|
||||
'import',
|
||||
'insights',
|
||||
'gsc',
|
||||
'logs',
|
||||
];
|
||||
}
|
||||
|
||||
@@ -221,6 +224,20 @@ export function bootWorkers() {
|
||||
logger.info('Started worker for gsc', { concurrency });
|
||||
}
|
||||
|
||||
// Start logs worker
|
||||
if (enabledQueues.includes('logs')) {
|
||||
const concurrency = getConcurrencyFor('logs', 10);
|
||||
const logsWorker = new Worker(
|
||||
logsQueue.name,
|
||||
async (job) => {
|
||||
await incomingLog(job.data.payload);
|
||||
},
|
||||
{ ...workerOptions, concurrency },
|
||||
);
|
||||
workers.push(logsWorker);
|
||||
logger.info('Started worker for logs', { concurrency });
|
||||
}
|
||||
|
||||
if (workers.length === 0) {
|
||||
logger.warn(
|
||||
'No workers started. Check ENABLED_QUEUES environment variable.'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Job } from 'bullmq';
|
||||
|
||||
import { eventBuffer, groupBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
|
||||
import { eventBuffer, groupBuffer, logBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
|
||||
import type { CronQueuePayload } from '@openpanel/queue';
|
||||
|
||||
import { jobdeleteProjects } from './cron.delete-projects';
|
||||
@@ -33,6 +33,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
|
||||
case 'flushGroups': {
|
||||
return await groupBuffer.tryFlush();
|
||||
}
|
||||
case 'flushLogs': {
|
||||
return await logBuffer.tryFlush();
|
||||
}
|
||||
case 'ping': {
|
||||
return await ping();
|
||||
}
|
||||
|
||||
63
apps/worker/src/jobs/logs.incoming-log.ts
Normal file
63
apps/worker/src/jobs/logs.incoming-log.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import type { IClickhouseLog } from '@openpanel/db';
|
||||
import { logBuffer } from '@openpanel/db';
|
||||
import type { LogsQueuePayload } from '@openpanel/queue';
|
||||
import { SEVERITY_TEXT_TO_NUMBER } from '@openpanel/validation';
|
||||
import { logger as baseLogger } from '@/utils/logger';
|
||||
|
||||
export async function incomingLog(
|
||||
payload: LogsQueuePayload['payload'],
|
||||
): Promise<void> {
|
||||
const logger = baseLogger.child({ projectId: payload.projectId });
|
||||
|
||||
try {
|
||||
const { log, uaInfo, geo, deviceId, sessionId, projectId, headers } = payload;
|
||||
|
||||
const sdkName = headers['openpanel-sdk-name'] ?? '';
|
||||
const sdkVersion = headers['openpanel-sdk-version'] ?? '';
|
||||
|
||||
const severityNumber =
|
||||
log.severityNumber ??
|
||||
SEVERITY_TEXT_TO_NUMBER[log.severity] ??
|
||||
9; // INFO fallback
|
||||
|
||||
const row: IClickhouseLog = {
|
||||
project_id: projectId,
|
||||
device_id: deviceId,
|
||||
profile_id: log.profileId ? String(log.profileId) : '',
|
||||
session_id: sessionId,
|
||||
timestamp: log.timestamp,
|
||||
observed_at: new Date().toISOString(),
|
||||
severity_number: severityNumber,
|
||||
severity_text: log.severity,
|
||||
body: log.body,
|
||||
trace_id: log.traceId ?? '',
|
||||
span_id: log.spanId ?? '',
|
||||
trace_flags: log.traceFlags ?? 0,
|
||||
logger_name: log.loggerName ?? '',
|
||||
attributes: log.attributes ?? {},
|
||||
resource: log.resource ?? {},
|
||||
sdk_name: sdkName,
|
||||
sdk_version: sdkVersion,
|
||||
country: geo.country ?? '',
|
||||
city: geo.city ?? '',
|
||||
region: geo.region ?? '',
|
||||
os: uaInfo.os ?? '',
|
||||
os_version: uaInfo.osVersion ?? '',
|
||||
browser: uaInfo.isServer ? '' : (uaInfo.browser ?? ''),
|
||||
browser_version: uaInfo.isServer ? '' : (uaInfo.browserVersion ?? ''),
|
||||
device: uaInfo.device ?? '',
|
||||
brand: uaInfo.isServer ? '' : (uaInfo.brand ?? ''),
|
||||
model: uaInfo.isServer ? '' : (uaInfo.model ?? ''),
|
||||
};
|
||||
|
||||
logBuffer.add(row);
|
||||
|
||||
logger.info('Log queued', {
|
||||
severity: log.severity,
|
||||
loggerName: log.loggerName,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to process incoming log', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
146
docker-compose.prod.yml
Normal file
146
docker-compose.prod.yml
Normal file
@@ -0,0 +1,146 @@
|
||||
services:
|
||||
op-db:
|
||||
image: postgres:18.3-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-db-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
environment:
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
op-kv:
|
||||
image: redis:8.6.2-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-kv-data:/data
|
||||
command: ["redis-server", "--maxmemory-policy", "noeviction"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
op-ch:
|
||||
image: clickhouse/clickhouse-server:26.3.2.3
|
||||
restart: always
|
||||
environment:
|
||||
- CLICKHOUSE_DEFAULT_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse}
|
||||
volumes:
|
||||
- op-ch-data:/var/lib/clickhouse
|
||||
- op-ch-logs:/var/log/clickhouse-server
|
||||
- ./self-hosting/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/op-config.xml:ro
|
||||
- ./self-hosting/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro
|
||||
- ./self-hosting/clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "clickhouse-client --query 'SELECT 1'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 262144
|
||||
hard: 262144
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
op-api:
|
||||
image: git.zias.be/zias/openpanel-api:latest
|
||||
restart: always
|
||||
ports:
|
||||
- "3001:3000"
|
||||
command: >
|
||||
sh -c "
|
||||
echo 'Running migrations...'
|
||||
CI=true pnpm -r run migrate:deploy
|
||||
|
||||
pnpm start
|
||||
"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
depends_on:
|
||||
op-db:
|
||||
condition: service_healthy
|
||||
op-ch:
|
||||
condition: service_healthy
|
||||
op-kv:
|
||||
condition: service_healthy
|
||||
env_file:
|
||||
- .env.prod
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
|
||||
op-dashboard:
|
||||
image: git.zias.be/zias/openpanel-dashboard:latest
|
||||
restart: always
|
||||
ports:
|
||||
- "3000:3000"
|
||||
depends_on:
|
||||
op-api:
|
||||
condition: service_healthy
|
||||
env_file:
|
||||
- .env.prod
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "3"
|
||||
|
||||
op-worker:
|
||||
image: git.zias.be/zias/openpanel-worker:latest
|
||||
restart: always
|
||||
ports:
|
||||
- "3002:3000"
|
||||
depends_on:
|
||||
op-api:
|
||||
condition: service_healthy
|
||||
env_file:
|
||||
- .env.prod
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "30m"
|
||||
max-file: "3"
|
||||
|
||||
volumes:
|
||||
op-db-data:
|
||||
driver: local
|
||||
op-kv-data:
|
||||
driver: local
|
||||
op-ch-data:
|
||||
driver: local
|
||||
op-ch-logs:
|
||||
driver: local
|
||||
@@ -2,7 +2,7 @@ version: "3"
|
||||
|
||||
services:
|
||||
op-db:
|
||||
image: postgres:14-alpine
|
||||
image: postgres:18.3-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- ./docker/data/op-db-data:/var/lib/postgresql/data
|
||||
@@ -13,7 +13,7 @@ services:
|
||||
- POSTGRES_PASSWORD=postgres
|
||||
|
||||
op-kv:
|
||||
image: redis:7.2.5-alpine
|
||||
image: redis:8.6.2-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- ./docker/data/op-kv-data:/data
|
||||
@@ -22,7 +22,7 @@ services:
|
||||
- 6379:6379
|
||||
|
||||
op-ch:
|
||||
image: clickhouse/clickhouse-server:26.1.3.52
|
||||
image: clickhouse/clickhouse-server:26.3.2.3
|
||||
restart: always
|
||||
volumes:
|
||||
- ./docker/data/op-ch-data:/var/lib/clickhouse
|
||||
|
||||
21
package.json
21
package.json
@@ -5,7 +5,7 @@
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"author": "Carl-Gerhard Lindesvärd",
|
||||
"packageManager": "pnpm@10.6.2",
|
||||
"packageManager": "pnpm@10.33.0",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"gen:bots": "pnpm -r --filter api gen:bots",
|
||||
@@ -30,11 +30,11 @@
|
||||
"pre-push": "[ -n \"$SKIP_HOOKS\" ] || (pnpm typecheck && pnpm test)"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"dotenv-cli": "^7.3.0",
|
||||
"semver": "^7.5.4",
|
||||
"@hyperdx/node-opentelemetry": "^0.10.3",
|
||||
"dotenv-cli": "^7.4.4",
|
||||
"semver": "^7.7.4",
|
||||
"typescript": "catalog:",
|
||||
"winston": "^3.14.2"
|
||||
"winston": "^3.19.0"
|
||||
},
|
||||
"trustedDependencies": [
|
||||
"@biomejs/biome",
|
||||
@@ -49,15 +49,16 @@
|
||||
"sharp"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.3.15",
|
||||
"@biomejs/biome": "2.4.10",
|
||||
"depcheck": "^1.4.7",
|
||||
"simple-git-hooks": "^2.12.1",
|
||||
"ultracite": "7.2.0",
|
||||
"vitest": "^3.0.4"
|
||||
"simple-git-hooks": "^2.13.1",
|
||||
"ultracite": "7.4.0",
|
||||
"vitest": "^3.2.4"
|
||||
},
|
||||
"pnpm": {
|
||||
"patchedDependencies": {
|
||||
"nuqs": "patches/nuqs.patch"
|
||||
"nuqs": "patches/nuqs.patch",
|
||||
"buffer-equal-constant-time": "patches/buffer-equal-constant-time.patch"
|
||||
},
|
||||
"overrides": {
|
||||
"rolldown": "1.0.0-beta.43",
|
||||
|
||||
@@ -12,13 +12,13 @@
|
||||
"@openpanel/validation": "workspace:^",
|
||||
"@oslojs/crypto": "^1.0.1",
|
||||
"@oslojs/encoding": "^1.1.0",
|
||||
"arctic": "^2.3.0"
|
||||
"arctic": "^2.3.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/react": "catalog:",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/constants": "workspace:*",
|
||||
"date-fns": "^3.3.1",
|
||||
"lru-cache": "^11.2.4",
|
||||
"date-fns": "^3.6.0",
|
||||
"lru-cache": "^11.2.7",
|
||||
"luxon": "^3.7.2",
|
||||
"mathjs": "^12.3.2",
|
||||
"nanoid": "^5.1.6",
|
||||
"ramda": "^0.29.1",
|
||||
"slugify": "^1.6.6",
|
||||
"mathjs": "^15.1.1",
|
||||
"nanoid": "^5.1.7",
|
||||
"ramda": "^0.32.0",
|
||||
"slugify": "^1.6.8",
|
||||
"superjson": "^1.13.3",
|
||||
"ua-parser-js": "^2.0.6",
|
||||
"ua-parser-js": "^2.0.9",
|
||||
"unique-names-generator": "^4.7.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -31,9 +31,9 @@
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@types/luxon": "^3.7.1",
|
||||
"@types/node": "catalog:",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/ua-parser-js": "^0.7.39",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"date-fns": "^3.3.1",
|
||||
"date-fns": "^3.6.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
72
packages/db/code-migrations/13-add-logs.ts
Normal file
72
packages/db/code-migrations/13-add-logs.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
import { createTable, runClickhouseMigrationCommands } from '../src/clickhouse/migration';
|
||||
import { getIsCluster, getIsSelfHosting, printBoxMessage } from './helpers';
|
||||
|
||||
export async function up() {
|
||||
const replicatedVersion = '1';
|
||||
const isClustered = getIsCluster();
|
||||
|
||||
const sqls: string[] = [];
|
||||
|
||||
sqls.push(
|
||||
...createTable({
|
||||
name: 'logs',
|
||||
columns: [
|
||||
'`id` UUID DEFAULT generateUUIDv4()',
|
||||
'`project_id` String CODEC(ZSTD(3))',
|
||||
'`device_id` String CODEC(ZSTD(3))',
|
||||
'`profile_id` String CODEC(ZSTD(3))',
|
||||
'`session_id` String CODEC(LZ4)',
|
||||
// OpenTelemetry log fields
|
||||
'`timestamp` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
|
||||
'`observed_at` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
|
||||
'`severity_number` UInt8',
|
||||
'`severity_text` LowCardinality(String)',
|
||||
'`body` String CODEC(ZSTD(3))',
|
||||
'`trace_id` String CODEC(ZSTD(3))',
|
||||
'`span_id` String CODEC(ZSTD(3))',
|
||||
'`trace_flags` UInt32 DEFAULT 0',
|
||||
'`logger_name` LowCardinality(String)',
|
||||
// OTel attributes (log-level key-value pairs)
|
||||
'`attributes` Map(String, String) CODEC(ZSTD(3))',
|
||||
// OTel resource attributes (device/app metadata)
|
||||
'`resource` Map(String, String) CODEC(ZSTD(3))',
|
||||
// Server-enriched context
|
||||
'`sdk_name` LowCardinality(String)',
|
||||
'`sdk_version` LowCardinality(String)',
|
||||
'`country` LowCardinality(FixedString(2))',
|
||||
'`city` String',
|
||||
'`region` LowCardinality(String)',
|
||||
'`os` LowCardinality(String)',
|
||||
'`os_version` LowCardinality(String)',
|
||||
'`browser` LowCardinality(String)',
|
||||
'`browser_version` LowCardinality(String)',
|
||||
'`device` LowCardinality(String)',
|
||||
'`brand` LowCardinality(String)',
|
||||
'`model` LowCardinality(String)',
|
||||
],
|
||||
indices: [
|
||||
'INDEX idx_severity_number severity_number TYPE minmax GRANULARITY 1',
|
||||
'INDEX idx_body body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1',
|
||||
'INDEX idx_trace_id trace_id TYPE bloom_filter GRANULARITY 1',
|
||||
'INDEX idx_logger_name logger_name TYPE bloom_filter GRANULARITY 1',
|
||||
],
|
||||
orderBy: ['project_id', 'toDate(timestamp)', 'severity_number', 'device_id'],
|
||||
partitionBy: 'toYYYYMM(timestamp)',
|
||||
settings: {
|
||||
index_granularity: 8192,
|
||||
ttl_only_drop_parts: 1,
|
||||
},
|
||||
distributionHash: 'cityHash64(project_id, toString(toStartOfHour(timestamp)))',
|
||||
replicatedVersion,
|
||||
isClustered,
|
||||
}),
|
||||
);
|
||||
|
||||
printBoxMessage('Running migration: 13-add-logs', [
|
||||
'Creates the logs table for OpenTelemetry-compatible device/app log capture.',
|
||||
]);
|
||||
|
||||
if (!process.argv.includes('--dry')) {
|
||||
await runClickhouseMigrationCommands(sqls);
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@
|
||||
"with-env": "dotenv -e ../../.env -c --"
|
||||
},
|
||||
"dependencies": {
|
||||
"@clickhouse/client": "^1.12.1",
|
||||
"@clickhouse/client": "^1.18.2",
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/constants": "workspace:*",
|
||||
"@openpanel/json": "workspace:*",
|
||||
@@ -21,13 +21,13 @@
|
||||
"@openpanel/queue": "workspace:^",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@prisma/client": "^6.14.0",
|
||||
"@prisma/extension-read-replicas": "^0.4.1",
|
||||
"@prisma/client": "^6.19.2",
|
||||
"@prisma/extension-read-replicas": "^0.5.0",
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"jiti": "^2.4.1",
|
||||
"mathjs": "^12.3.2",
|
||||
"prisma-json-types-generator": "^3.1.1",
|
||||
"ramda": "^0.29.1",
|
||||
"jiti": "^2.6.1",
|
||||
"mathjs": "^15.1.1",
|
||||
"prisma-json-types-generator": "^3.6.2",
|
||||
"ramda": "^0.32.0",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"uuid": "^9.0.1",
|
||||
@@ -36,10 +36,10 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"prisma": "^6.14.0",
|
||||
"@types/uuid": "^11.0.0",
|
||||
"prisma": "^6.19.2",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { getRedisCache } from '@openpanel/redis';
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { ch } from '../clickhouse/client';
|
||||
import * as chClient from '../clickhouse/client';
|
||||
const { ch } = chClient;
|
||||
|
||||
// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
|
||||
vi.mock('../services/event.service', () => ({}));
|
||||
@@ -10,7 +11,8 @@ import { EventBuffer } from './event-buffer';
|
||||
const redis = getRedisCache();
|
||||
|
||||
beforeEach(async () => {
|
||||
await redis.flushdb();
|
||||
const keys = await redis.keys('event*');
|
||||
if (keys.length > 0) await redis.del(...keys);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -209,18 +211,16 @@ describe('EventBuffer', () => {
|
||||
});
|
||||
|
||||
it('tracks active visitors', async () => {
|
||||
const event = {
|
||||
project_id: 'p9',
|
||||
profile_id: 'u9',
|
||||
name: 'custom',
|
||||
created_at: new Date().toISOString(),
|
||||
} as any;
|
||||
|
||||
eventBuffer.add(event);
|
||||
await eventBuffer.flush();
|
||||
const querySpy = vi
|
||||
.spyOn(chClient, 'chQuery')
|
||||
.mockResolvedValueOnce([{ count: 2 }] as any);
|
||||
|
||||
const count = await eventBuffer.getActiveVisitorCount('p9');
|
||||
expect(count).toBeGreaterThanOrEqual(1);
|
||||
expect(count).toBe(2);
|
||||
expect(querySpy).toHaveBeenCalledOnce();
|
||||
expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");
|
||||
|
||||
querySpy.mockRestore();
|
||||
});
|
||||
|
||||
it('handles multiple sessions independently — all events go to buffer', async () => {
|
||||
@@ -273,4 +273,24 @@ describe('EventBuffer', () => {
|
||||
|
||||
expect(await eventBuffer.getBufferSize()).toBe(5);
|
||||
});
|
||||
|
||||
it('retains events in queue when ClickHouse insert fails', async () => {
|
||||
eventBuffer.add({
|
||||
project_id: 'p12',
|
||||
name: 'event1',
|
||||
created_at: new Date().toISOString(),
|
||||
} as any);
|
||||
await eventBuffer.flush();
|
||||
|
||||
const insertSpy = vi
|
||||
.spyOn(ch, 'insert')
|
||||
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
|
||||
|
||||
await eventBuffer.processBuffer();
|
||||
|
||||
// Events must still be in the queue — not lost
|
||||
expect(await eventBuffer.getBufferSize()).toBe(1);
|
||||
|
||||
insertSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { getSafeJson } from '@openpanel/json';
|
||||
import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';
|
||||
import { ch } from '../clickhouse/client';
|
||||
import { getRedisCache, publishEvent } from '@openpanel/redis';
|
||||
import { ch, chQuery } from '../clickhouse/client';
|
||||
import type { IClickhouseEvent } from '../services/event.service';
|
||||
import { BaseBuffer } from './base-buffer';
|
||||
|
||||
@@ -25,10 +25,6 @@ export class EventBuffer extends BaseBuffer {
|
||||
/** Tracks consecutive flush failures for observability; reset on success. */
|
||||
private flushRetryCount = 0;
|
||||
|
||||
private activeVisitorsExpiration = 60 * 5; // 5 minutes
|
||||
/** How often (ms) we refresh the heartbeat key + zadd per visitor. */
|
||||
private heartbeatRefreshMs = 60_000; // 1 minute
|
||||
private lastHeartbeat = new Map<string, number>();
|
||||
private queueKey = 'event_buffer:queue';
|
||||
protected bufferCounterKey = 'event_buffer:total_count';
|
||||
|
||||
@@ -87,20 +83,12 @@ export class EventBuffer extends BaseBuffer {
|
||||
|
||||
for (const event of eventsToFlush) {
|
||||
multi.rpush(this.queueKey, JSON.stringify(event));
|
||||
if (event.profile_id) {
|
||||
this.incrementActiveVisitorCount(
|
||||
multi,
|
||||
event.project_id,
|
||||
event.profile_id
|
||||
);
|
||||
}
|
||||
}
|
||||
multi.incrby(this.bufferCounterKey, eventsToFlush.length);
|
||||
|
||||
await multi.exec();
|
||||
|
||||
this.flushRetryCount = 0;
|
||||
this.pruneHeartbeatMap();
|
||||
} catch (error) {
|
||||
// Re-queue failed events at the front to preserve order and avoid data loss
|
||||
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
|
||||
@@ -202,58 +190,21 @@ export class EventBuffer extends BaseBuffer {
|
||||
}
|
||||
}
|
||||
|
||||
public async getBufferSize() {
|
||||
public getBufferSize() {
|
||||
return this.getBufferSizeWithCounter(async () => {
|
||||
const redis = getRedisCache();
|
||||
return await redis.llen(this.queueKey);
|
||||
});
|
||||
}
|
||||
|
||||
private pruneHeartbeatMap() {
|
||||
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
|
||||
for (const [key, ts] of this.lastHeartbeat) {
|
||||
if (ts < cutoff) {
|
||||
this.lastHeartbeat.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private incrementActiveVisitorCount(
|
||||
multi: ReturnType<Redis['multi']>,
|
||||
projectId: string,
|
||||
profileId: string
|
||||
) {
|
||||
const key = `${projectId}:${profileId}`;
|
||||
const now = Date.now();
|
||||
const last = this.lastHeartbeat.get(key) ?? 0;
|
||||
|
||||
if (now - last < this.heartbeatRefreshMs) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.lastHeartbeat.set(key, now);
|
||||
const zsetKey = `live:visitors:${projectId}`;
|
||||
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
|
||||
multi
|
||||
.zadd(zsetKey, now, profileId)
|
||||
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
|
||||
}
|
||||
|
||||
public async getActiveVisitorCount(projectId: string): Promise<number> {
|
||||
const redis = getRedisCache();
|
||||
const zsetKey = `live:visitors:${projectId}`;
|
||||
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
|
||||
|
||||
const multi = redis.multi();
|
||||
multi
|
||||
.zremrangebyscore(zsetKey, '-inf', cutoff)
|
||||
.zcount(zsetKey, cutoff, '+inf');
|
||||
|
||||
const [, count] = (await multi.exec()) as [
|
||||
[Error | null, any],
|
||||
[Error | null, number],
|
||||
];
|
||||
|
||||
return count[1] || 0;
|
||||
const rows = await chQuery<{ count: number }>(
|
||||
`SELECT uniq(profile_id) AS count
|
||||
FROM events
|
||||
WHERE project_id = '${projectId}'
|
||||
AND profile_id != ''
|
||||
AND created_at >= now() - INTERVAL 5 MINUTE`
|
||||
);
|
||||
return rows[0]?.count ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { BotBuffer as BotBufferRedis } from './bot-buffer';
|
||||
import { EventBuffer as EventBufferRedis } from './event-buffer';
|
||||
import { GroupBuffer } from './group-buffer';
|
||||
import { LogBuffer } from './log-buffer';
|
||||
import { ProfileBackfillBuffer } from './profile-backfill-buffer';
|
||||
import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
|
||||
import { ReplayBuffer } from './replay-buffer';
|
||||
@@ -13,6 +14,8 @@ export const sessionBuffer = new SessionBuffer();
|
||||
export const profileBackfillBuffer = new ProfileBackfillBuffer();
|
||||
export const replayBuffer = new ReplayBuffer();
|
||||
export const groupBuffer = new GroupBuffer();
|
||||
export const logBuffer = new LogBuffer();
|
||||
|
||||
export type { ProfileBackfillEntry } from './profile-backfill-buffer';
|
||||
export type { IClickhouseSessionReplayChunk } from './replay-buffer';
|
||||
export type { IClickhouseLog } from './log-buffer';
|
||||
|
||||
193
packages/db/src/buffers/log-buffer.ts
Normal file
193
packages/db/src/buffers/log-buffer.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
import { getSafeJson } from '@openpanel/json';
|
||||
import { getRedisCache } from '@openpanel/redis';
|
||||
import { ch } from '../clickhouse/client';
|
||||
import { BaseBuffer } from './base-buffer';
|
||||
|
||||
export interface IClickhouseLog {
|
||||
id?: string;
|
||||
project_id: string;
|
||||
device_id: string;
|
||||
profile_id: string;
|
||||
session_id: string;
|
||||
timestamp: string;
|
||||
observed_at: string;
|
||||
severity_number: number;
|
||||
severity_text: string;
|
||||
body: string;
|
||||
trace_id: string;
|
||||
span_id: string;
|
||||
trace_flags: number;
|
||||
logger_name: string;
|
||||
attributes: Record<string, string>;
|
||||
resource: Record<string, string>;
|
||||
sdk_name: string;
|
||||
sdk_version: string;
|
||||
country: string;
|
||||
city: string;
|
||||
region: string;
|
||||
os: string;
|
||||
os_version: string;
|
||||
browser: string;
|
||||
browser_version: string;
|
||||
device: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export class LogBuffer extends BaseBuffer {
|
||||
private batchSize = process.env.LOG_BUFFER_BATCH_SIZE
|
||||
? Number.parseInt(process.env.LOG_BUFFER_BATCH_SIZE, 10)
|
||||
: 4000;
|
||||
private chunkSize = process.env.LOG_BUFFER_CHUNK_SIZE
|
||||
? Number.parseInt(process.env.LOG_BUFFER_CHUNK_SIZE, 10)
|
||||
: 1000;
|
||||
private microBatchIntervalMs = process.env.LOG_BUFFER_MICRO_BATCH_MS
|
||||
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_MS, 10)
|
||||
: 10;
|
||||
private microBatchMaxSize = process.env.LOG_BUFFER_MICRO_BATCH_SIZE
|
||||
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_SIZE, 10)
|
||||
: 100;
|
||||
|
||||
private pendingLogs: IClickhouseLog[] = [];
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private isFlushing = false;
|
||||
private flushRetryCount = 0;
|
||||
|
||||
private queueKey = 'log_buffer:queue';
|
||||
protected bufferCounterKey = 'log_buffer:total_count';
|
||||
|
||||
constructor() {
|
||||
super({
|
||||
name: 'log',
|
||||
onFlush: async () => {
|
||||
await this.processBuffer();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
add(log: IClickhouseLog) {
|
||||
this.pendingLogs.push(log);
|
||||
|
||||
if (this.pendingLogs.length >= this.microBatchMaxSize) {
|
||||
this.flushLocalBuffer();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.flushTimer) {
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flushTimer = null;
|
||||
this.flushLocalBuffer();
|
||||
}, this.microBatchIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
public async flush() {
|
||||
if (this.flushTimer) {
|
||||
clearTimeout(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
await this.flushLocalBuffer();
|
||||
}
|
||||
|
||||
private async flushLocalBuffer() {
|
||||
if (this.isFlushing || this.pendingLogs.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isFlushing = true;
|
||||
const logsToFlush = this.pendingLogs;
|
||||
this.pendingLogs = [];
|
||||
|
||||
try {
|
||||
const redis = getRedisCache();
|
||||
const multi = redis.multi();
|
||||
for (const log of logsToFlush) {
|
||||
multi.rpush(this.queueKey, JSON.stringify(log));
|
||||
}
|
||||
multi.incrby(this.bufferCounterKey, logsToFlush.length);
|
||||
await multi.exec();
|
||||
this.flushRetryCount = 0;
|
||||
} catch (error) {
|
||||
this.pendingLogs = logsToFlush.concat(this.pendingLogs);
|
||||
this.flushRetryCount += 1;
|
||||
this.logger.warn('Failed to flush log buffer to Redis; logs re-queued', {
|
||||
error,
|
||||
logCount: logsToFlush.length,
|
||||
flushRetryCount: this.flushRetryCount,
|
||||
});
|
||||
} finally {
|
||||
this.isFlushing = false;
|
||||
if (this.pendingLogs.length > 0 && !this.flushTimer) {
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flushTimer = null;
|
||||
this.flushLocalBuffer();
|
||||
}, this.microBatchIntervalMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async processBuffer() {
|
||||
const redis = getRedisCache();
|
||||
|
||||
try {
|
||||
const queueLogs = await redis.lrange(this.queueKey, 0, this.batchSize - 1);
|
||||
|
||||
if (queueLogs.length === 0) {
|
||||
this.logger.debug('No logs to process');
|
||||
return;
|
||||
}
|
||||
|
||||
const logsToClickhouse: IClickhouseLog[] = [];
|
||||
for (const logStr of queueLogs) {
|
||||
const log = getSafeJson<IClickhouseLog>(logStr);
|
||||
if (log) {
|
||||
logsToClickhouse.push(log);
|
||||
}
|
||||
}
|
||||
|
||||
if (logsToClickhouse.length === 0) {
|
||||
this.logger.debug('No valid logs to process');
|
||||
return;
|
||||
}
|
||||
|
||||
logsToClickhouse.sort(
|
||||
(a, b) =>
|
||||
new Date(a.timestamp || 0).getTime() -
|
||||
new Date(b.timestamp || 0).getTime(),
|
||||
);
|
||||
|
||||
this.logger.info('Inserting logs into ClickHouse', {
|
||||
totalLogs: logsToClickhouse.length,
|
||||
chunks: Math.ceil(logsToClickhouse.length / this.chunkSize),
|
||||
});
|
||||
|
||||
for (const chunk of this.chunks(logsToClickhouse, this.chunkSize)) {
|
||||
await ch.insert({
|
||||
table: 'logs',
|
||||
values: chunk,
|
||||
format: 'JSONEachRow',
|
||||
});
|
||||
}
|
||||
|
||||
await redis
|
||||
.multi()
|
||||
.ltrim(this.queueKey, queueLogs.length, -1)
|
||||
.decrby(this.bufferCounterKey, queueLogs.length)
|
||||
.exec();
|
||||
|
||||
this.logger.info('Processed logs from Redis buffer', {
|
||||
batchSize: this.batchSize,
|
||||
logsProcessed: logsToClickhouse.length,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('Error processing log Redis buffer', { error });
|
||||
}
|
||||
}
|
||||
|
||||
public getBufferSize() {
|
||||
return this.getBufferSizeWithCounter(async () => {
|
||||
const redis = getRedisCache();
|
||||
return await redis.llen(this.queueKey);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { getRedisCache } from '@openpanel/redis';
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { getSafeJson } from '@openpanel/json';
|
||||
import type { IClickhouseProfile } from '../services/profile.service';
|
||||
|
||||
// Mock chQuery to avoid hitting real ClickHouse
|
||||
@@ -36,7 +35,11 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await redis.flushdb();
|
||||
const keys = [
|
||||
...await redis.keys('profile*'),
|
||||
...await redis.keys('lock:profile'),
|
||||
];
|
||||
if (keys.length > 0) await redis.del(...keys);
|
||||
vi.mocked(chQuery).mockResolvedValue([]);
|
||||
});
|
||||
|
||||
@@ -63,64 +66,12 @@ describe('ProfileBuffer', () => {
|
||||
expect(sizeAfter).toBe(sizeBefore + 1);
|
||||
});
|
||||
|
||||
it('merges subsequent updates via cache (sequential calls)', async () => {
|
||||
it('concurrent adds: both raw profiles are queued', async () => {
|
||||
const identifyProfile = makeProfile({
|
||||
first_name: 'John',
|
||||
email: 'john@example.com',
|
||||
groups: [],
|
||||
});
|
||||
|
||||
const groupProfile = makeProfile({
|
||||
first_name: '',
|
||||
email: '',
|
||||
groups: ['group-abc'],
|
||||
});
|
||||
|
||||
// Sequential: identify first, then group
|
||||
await profileBuffer.add(identifyProfile);
|
||||
await profileBuffer.add(groupProfile);
|
||||
|
||||
// Second add should read the cached identify profile and merge groups in
|
||||
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
|
||||
expect(cached?.first_name).toBe('John');
|
||||
expect(cached?.email).toBe('john@example.com');
|
||||
expect(cached?.groups).toContain('group-abc');
|
||||
});
|
||||
|
||||
it('race condition: concurrent identify + group calls preserve all data', async () => {
|
||||
const identifyProfile = makeProfile({
|
||||
first_name: 'John',
|
||||
email: 'john@example.com',
|
||||
groups: [],
|
||||
});
|
||||
|
||||
const groupProfile = makeProfile({
|
||||
first_name: '',
|
||||
email: '',
|
||||
groups: ['group-abc'],
|
||||
});
|
||||
|
||||
// Both calls run concurrently — the per-profile lock serializes them so the
|
||||
// second one reads the first's result from cache and merges correctly.
|
||||
await Promise.all([
|
||||
profileBuffer.add(identifyProfile),
|
||||
profileBuffer.add(groupProfile),
|
||||
]);
|
||||
|
||||
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
|
||||
|
||||
expect(cached?.first_name).toBe('John');
|
||||
expect(cached?.email).toBe('john@example.com');
|
||||
expect(cached?.groups).toContain('group-abc');
|
||||
});
|
||||
|
||||
it('race condition: concurrent writes produce one merged buffer entry', async () => {
|
||||
const identifyProfile = makeProfile({
|
||||
first_name: 'John',
|
||||
email: 'john@example.com',
|
||||
groups: [],
|
||||
});
|
||||
|
||||
const groupProfile = makeProfile({
|
||||
first_name: '',
|
||||
email: '',
|
||||
@@ -128,24 +79,126 @@ describe('ProfileBuffer', () => {
|
||||
});
|
||||
|
||||
const sizeBefore = await profileBuffer.getBufferSize();
|
||||
await Promise.all([
|
||||
profileBuffer.add(identifyProfile),
|
||||
profileBuffer.add(groupProfile),
|
||||
]);
|
||||
const sizeAfter = await profileBuffer.getBufferSize();
|
||||
|
||||
// Both raw profiles are queued; merge happens at flush time
|
||||
expect(sizeAfter).toBe(sizeBefore + 2);
|
||||
});
|
||||
|
||||
it('merges sequential updates for the same profile at flush time', async () => {
|
||||
const identifyProfile = makeProfile({
|
||||
first_name: 'John',
|
||||
email: 'john@example.com',
|
||||
groups: [],
|
||||
});
|
||||
const groupProfile = makeProfile({
|
||||
first_name: '',
|
||||
email: '',
|
||||
groups: ['group-abc'],
|
||||
});
|
||||
|
||||
await profileBuffer.add(identifyProfile);
|
||||
await profileBuffer.add(groupProfile);
|
||||
await profileBuffer.processBuffer();
|
||||
|
||||
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
|
||||
expect(cached?.first_name).toBe('John');
|
||||
expect(cached?.email).toBe('john@example.com');
|
||||
expect(cached?.groups).toContain('group-abc');
|
||||
});
|
||||
|
||||
it('merges concurrent updates for the same profile at flush time', async () => {
|
||||
const identifyProfile = makeProfile({
|
||||
first_name: 'John',
|
||||
email: 'john@example.com',
|
||||
groups: [],
|
||||
});
|
||||
const groupProfile = makeProfile({
|
||||
first_name: '',
|
||||
email: '',
|
||||
groups: ['group-abc'],
|
||||
});
|
||||
|
||||
await Promise.all([
|
||||
profileBuffer.add(identifyProfile),
|
||||
profileBuffer.add(groupProfile),
|
||||
]);
|
||||
await profileBuffer.processBuffer();
|
||||
|
||||
const sizeAfter = await profileBuffer.getBufferSize();
|
||||
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
|
||||
expect(cached?.first_name).toBe('John');
|
||||
expect(cached?.email).toBe('john@example.com');
|
||||
expect(cached?.groups).toContain('group-abc');
|
||||
});
|
||||
|
||||
// The second add merges into the first — only 2 buffer entries total
|
||||
// (one from identify, one merged update with group)
|
||||
expect(sizeAfter).toBe(sizeBefore + 2);
|
||||
it('uses existing ClickHouse data for cache misses when merging', async () => {
|
||||
const existingInClickhouse = makeProfile({
|
||||
first_name: 'Jane',
|
||||
email: 'jane@example.com',
|
||||
groups: ['existing-group'],
|
||||
});
|
||||
vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);
|
||||
|
||||
// The last entry in the buffer should have both name and group
|
||||
const rawEntries = await redis.lrange('profile-buffer', 0, -1);
|
||||
const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e));
|
||||
const lastEntry = entries[entries.length - 1];
|
||||
const incomingProfile = makeProfile({
|
||||
first_name: '',
|
||||
email: '',
|
||||
groups: ['new-group'],
|
||||
});
|
||||
|
||||
expect(lastEntry?.first_name).toBe('John');
|
||||
expect(lastEntry?.groups).toContain('group-abc');
|
||||
await profileBuffer.add(incomingProfile);
|
||||
await profileBuffer.processBuffer();
|
||||
|
||||
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
|
||||
expect(cached?.first_name).toBe('Jane');
|
||||
expect(cached?.email).toBe('jane@example.com');
|
||||
expect(cached?.groups).toContain('existing-group');
|
||||
expect(cached?.groups).toContain('new-group');
|
||||
});
|
||||
|
||||
it('buffer is empty after flush', async () => {
|
||||
await profileBuffer.add(makeProfile({ first_name: 'John' }));
|
||||
expect(await profileBuffer.getBufferSize()).toBe(1);
|
||||
|
||||
await profileBuffer.processBuffer();
|
||||
|
||||
expect(await profileBuffer.getBufferSize()).toBe(0);
|
||||
});
|
||||
|
||||
it('retains profiles in queue when ClickHouse insert fails', async () => {
|
||||
await profileBuffer.add(makeProfile({ first_name: 'John' }));
|
||||
|
||||
const { ch } = await import('../clickhouse/client');
|
||||
const insertSpy = vi
|
||||
.spyOn(ch, 'insert')
|
||||
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
|
||||
|
||||
await profileBuffer.processBuffer();
|
||||
|
||||
// Profiles must still be in the queue — not lost
|
||||
expect(await profileBuffer.getBufferSize()).toBe(1);
|
||||
|
||||
insertSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {
|
||||
vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));
|
||||
|
||||
const { ch } = await import('../clickhouse/client');
|
||||
const insertSpy = vi
|
||||
.spyOn(ch, 'insert')
|
||||
.mockResolvedValueOnce(undefined as any);
|
||||
|
||||
await profileBuffer.add(makeProfile({ first_name: 'John' }));
|
||||
await profileBuffer.processBuffer();
|
||||
|
||||
// Insert must still have been called — no data loss even when fetch fails
|
||||
expect(insertSpy).toHaveBeenCalled();
|
||||
expect(await profileBuffer.getBufferSize()).toBe(0);
|
||||
|
||||
insertSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
import { deepMergeObjects } from '@openpanel/common';
|
||||
import { generateSecureId } from '@openpanel/common/server';
|
||||
import { getSafeJson } from '@openpanel/json';
|
||||
import type { ILogger } from '@openpanel/logger';
|
||||
import { getRedisCache, type Redis } from '@openpanel/redis';
|
||||
import shallowEqual from 'fast-deep-equal';
|
||||
import { omit, uniq } from 'ramda';
|
||||
import sqlstring from 'sqlstring';
|
||||
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
|
||||
@@ -11,29 +8,24 @@ import type { IClickhouseProfile } from '../services/profile.service';
|
||||
import { BaseBuffer } from './base-buffer';
|
||||
|
||||
export class ProfileBuffer extends BaseBuffer {
|
||||
private batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
|
||||
private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
|
||||
? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)
|
||||
: 200;
|
||||
private chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
|
||||
private readonly chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
|
||||
? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10)
|
||||
: 1000;
|
||||
private ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
|
||||
private readonly ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
|
||||
? Number.parseInt(process.env.PROFILE_BUFFER_TTL_IN_SECONDS, 10)
|
||||
: 60 * 60;
|
||||
/** Max profiles per ClickHouse IN-clause fetch to keep query size bounded */
|
||||
private readonly fetchChunkSize = process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE
|
||||
? Number.parseInt(process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE, 10)
|
||||
: 50;
|
||||
|
||||
private readonly redisKey = 'profile-buffer';
|
||||
private readonly redisProfilePrefix = 'profile-cache:';
|
||||
|
||||
private redis: Redis;
|
||||
private releaseLockSha: string | null = null;
|
||||
|
||||
private readonly releaseLockScript = `
|
||||
if redis.call("get", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("del", KEYS[1])
|
||||
else
|
||||
return 0
|
||||
end
|
||||
`;
|
||||
private readonly redis: Redis;
|
||||
|
||||
constructor() {
|
||||
super({
|
||||
@@ -43,9 +35,6 @@ export class ProfileBuffer extends BaseBuffer {
|
||||
},
|
||||
});
|
||||
this.redis = getRedisCache();
|
||||
this.redis.script('LOAD', this.releaseLockScript).then((sha) => {
|
||||
this.releaseLockSha = sha as string;
|
||||
});
|
||||
}
|
||||
|
||||
private getProfileCacheKey({
|
||||
@@ -58,243 +47,236 @@ export class ProfileBuffer extends BaseBuffer {
|
||||
return `${this.redisProfilePrefix}${projectId}:${profileId}`;
|
||||
}
|
||||
|
||||
private async withProfileLock<T>(
|
||||
public async fetchFromCache(
|
||||
profileId: string,
|
||||
projectId: string,
|
||||
fn: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const lockKey = `profile-lock:${projectId}:${profileId}`;
|
||||
const lockId = generateSecureId('lock');
|
||||
const maxRetries = 20;
|
||||
const retryDelayMs = 50;
|
||||
|
||||
for (let i = 0; i < maxRetries; i++) {
|
||||
const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');
|
||||
if (acquired === 'OK') {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
if (this.releaseLockSha) {
|
||||
await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);
|
||||
} else {
|
||||
await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);
|
||||
}
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
|
||||
projectId: string
|
||||
): Promise<IClickhouseProfile | null> {
|
||||
const cacheKey = this.getProfileCacheKey({ profileId, projectId });
|
||||
const cached = await this.redis.get(cacheKey);
|
||||
if (!cached) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.logger.error(
|
||||
'Failed to acquire profile lock, proceeding without lock',
|
||||
{
|
||||
profileId,
|
||||
projectId,
|
||||
}
|
||||
);
|
||||
return fn();
|
||||
}
|
||||
|
||||
async alreadyExists(profile: IClickhouseProfile) {
|
||||
const cacheKey = this.getProfileCacheKey({
|
||||
profileId: profile.id,
|
||||
projectId: profile.project_id,
|
||||
});
|
||||
return (await this.redis.exists(cacheKey)) === 1;
|
||||
return getSafeJson<IClickhouseProfile>(cached);
|
||||
}
|
||||
|
||||
async add(profile: IClickhouseProfile, isFromEvent = false) {
|
||||
const logger = this.logger.child({
|
||||
projectId: profile.project_id,
|
||||
profileId: profile.id,
|
||||
});
|
||||
|
||||
try {
|
||||
logger.debug('Adding profile');
|
||||
|
||||
if (isFromEvent && (await this.alreadyExists(profile))) {
|
||||
logger.debug('Profile already created, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
await this.withProfileLock(profile.id, profile.project_id, async () => {
|
||||
const existingProfile = await this.fetchProfile(profile, logger);
|
||||
|
||||
// Delete any properties that are not server related if we have a non-server profile
|
||||
if (
|
||||
existingProfile?.properties.device !== 'server' &&
|
||||
profile.properties.device === 'server'
|
||||
) {
|
||||
profile.properties = omit(
|
||||
[
|
||||
'city',
|
||||
'country',
|
||||
'region',
|
||||
'longitude',
|
||||
'latitude',
|
||||
'os',
|
||||
'osVersion',
|
||||
'browser',
|
||||
'device',
|
||||
'isServer',
|
||||
'os_version',
|
||||
'browser_version',
|
||||
],
|
||||
profile.properties
|
||||
);
|
||||
}
|
||||
|
||||
const mergedProfile: IClickhouseProfile = existingProfile
|
||||
? {
|
||||
...deepMergeObjects(
|
||||
existingProfile,
|
||||
omit(['created_at', 'groups'], profile)
|
||||
),
|
||||
groups: uniq([
|
||||
...(existingProfile.groups ?? []),
|
||||
...(profile.groups ?? []),
|
||||
]),
|
||||
}
|
||||
: profile;
|
||||
|
||||
if (
|
||||
profile &&
|
||||
existingProfile &&
|
||||
shallowEqual(
|
||||
omit(['created_at'], existingProfile),
|
||||
omit(['created_at'], mergedProfile)
|
||||
)
|
||||
) {
|
||||
this.logger.debug('Profile not changed, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.debug('Merged profile will be inserted', {
|
||||
mergedProfile,
|
||||
existingProfile,
|
||||
profile,
|
||||
});
|
||||
|
||||
if (isFromEvent) {
|
||||
const cacheKey = this.getProfileCacheKey({
|
||||
profileId: profile.id,
|
||||
projectId: profile.project_id,
|
||||
});
|
||||
|
||||
const result = await this.redis
|
||||
.multi()
|
||||
.set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
|
||||
.rpush(this.redisKey, JSON.stringify(mergedProfile))
|
||||
.incr(this.bufferCounterKey)
|
||||
.llen(this.redisKey)
|
||||
.exec();
|
||||
|
||||
if (!result) {
|
||||
this.logger.error('Failed to add profile to Redis', {
|
||||
profile,
|
||||
cacheKey,
|
||||
});
|
||||
const exists = await this.redis.exists(cacheKey);
|
||||
if (exists === 1) {
|
||||
return;
|
||||
}
|
||||
const bufferLength = (result?.[3]?.[1] as number) ?? 0;
|
||||
}
|
||||
|
||||
this.logger.debug('Current buffer length', {
|
||||
bufferLength,
|
||||
batchSize: this.batchSize,
|
||||
});
|
||||
if (bufferLength >= this.batchSize) {
|
||||
await this.tryFlush();
|
||||
}
|
||||
});
|
||||
const result = await this.redis
|
||||
.multi()
|
||||
.rpush(this.redisKey, JSON.stringify(profile))
|
||||
.incr(this.bufferCounterKey)
|
||||
.llen(this.redisKey)
|
||||
.exec();
|
||||
|
||||
if (!result) {
|
||||
this.logger.error('Failed to add profile to Redis', { profile });
|
||||
return;
|
||||
}
|
||||
|
||||
const bufferLength = (result?.[2]?.[1] as number) ?? 0;
|
||||
if (bufferLength >= this.batchSize) {
|
||||
await this.tryFlush();
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to add profile', { error, profile });
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchProfile(
|
||||
profile: IClickhouseProfile,
|
||||
logger: ILogger
|
||||
): Promise<IClickhouseProfile | null> {
|
||||
const existingProfile = await this.fetchFromCache(
|
||||
profile.id,
|
||||
profile.project_id
|
||||
);
|
||||
if (existingProfile) {
|
||||
logger.debug('Profile found in Redis');
|
||||
return existingProfile;
|
||||
private mergeProfiles(
|
||||
existing: IClickhouseProfile | null,
|
||||
incoming: IClickhouseProfile
|
||||
): IClickhouseProfile {
|
||||
if (!existing) {
|
||||
return incoming;
|
||||
}
|
||||
|
||||
return this.fetchFromClickhouse(profile, logger);
|
||||
}
|
||||
|
||||
public async fetchFromCache(
|
||||
profileId: string,
|
||||
projectId: string
|
||||
): Promise<IClickhouseProfile | null> {
|
||||
const cacheKey = this.getProfileCacheKey({
|
||||
profileId,
|
||||
projectId,
|
||||
});
|
||||
const existingProfile = await this.redis.get(cacheKey);
|
||||
if (!existingProfile) {
|
||||
return null;
|
||||
let profile = incoming;
|
||||
if (
|
||||
existing.properties.device !== 'server' &&
|
||||
incoming.properties.device === 'server'
|
||||
) {
|
||||
profile = {
|
||||
...incoming,
|
||||
properties: omit(
|
||||
[
|
||||
'city',
|
||||
'country',
|
||||
'region',
|
||||
'longitude',
|
||||
'latitude',
|
||||
'os',
|
||||
'osVersion',
|
||||
'browser',
|
||||
'device',
|
||||
'isServer',
|
||||
'os_version',
|
||||
'browser_version',
|
||||
],
|
||||
incoming.properties
|
||||
),
|
||||
};
|
||||
}
|
||||
return getSafeJson<IClickhouseProfile>(existingProfile);
|
||||
|
||||
return {
|
||||
...deepMergeObjects(existing, omit(['created_at', 'groups'], profile)),
|
||||
groups: uniq([...(existing.groups ?? []), ...(incoming.groups ?? [])]),
|
||||
};
|
||||
}
|
||||
|
||||
private async fetchFromClickhouse(
|
||||
profile: IClickhouseProfile,
|
||||
logger: ILogger
|
||||
): Promise<IClickhouseProfile | null> {
|
||||
logger.debug('Fetching profile from Clickhouse');
|
||||
const result = await chQuery<IClickhouseProfile>(
|
||||
`SELECT
|
||||
id,
|
||||
project_id,
|
||||
last_value(nullIf(first_name, '')) as first_name,
|
||||
last_value(nullIf(last_name, '')) as last_name,
|
||||
last_value(nullIf(email, '')) as email,
|
||||
last_value(nullIf(avatar, '')) as avatar,
|
||||
last_value(is_external) as is_external,
|
||||
last_value(properties) as properties,
|
||||
last_value(created_at) as created_at
|
||||
FROM ${TABLE_NAMES.profiles}
|
||||
WHERE
|
||||
id = ${sqlstring.escape(String(profile.id))} AND
|
||||
project_id = ${sqlstring.escape(profile.project_id)}
|
||||
${
|
||||
profile.is_external === false
|
||||
? ' AND profiles.created_at > now() - INTERVAL 2 DAY'
|
||||
: ''
|
||||
private async batchFetchFromClickhouse(
|
||||
profiles: IClickhouseProfile[]
|
||||
): Promise<Map<string, IClickhouseProfile>> {
|
||||
const result = new Map<string, IClickhouseProfile>();
|
||||
|
||||
// Non-external (anonymous/device) profiles get a 2-day recency filter to
|
||||
// avoid pulling stale anonymous sessions from far back.
|
||||
const external = profiles.filter((p) => p.is_external !== false);
|
||||
const nonExternal = profiles.filter((p) => p.is_external === false);
|
||||
|
||||
const fetchGroup = async (
|
||||
group: IClickhouseProfile[],
|
||||
withDateFilter: boolean
|
||||
) => {
|
||||
for (const chunk of this.chunks(group, this.fetchChunkSize)) {
|
||||
const tuples = chunk
|
||||
.map(
|
||||
(p) =>
|
||||
`(${sqlstring.escape(String(p.id))}, ${sqlstring.escape(p.project_id)})`
|
||||
)
|
||||
.join(', ');
|
||||
try {
|
||||
const rows = await chQuery<IClickhouseProfile>(
|
||||
`SELECT
|
||||
id,
|
||||
project_id,
|
||||
argMax(nullIf(first_name, ''), ${TABLE_NAMES.profiles}.created_at) as first_name,
|
||||
argMax(nullIf(last_name, ''), ${TABLE_NAMES.profiles}.created_at) as last_name,
|
||||
argMax(nullIf(email, ''), ${TABLE_NAMES.profiles}.created_at) as email,
|
||||
argMax(nullIf(avatar, ''), ${TABLE_NAMES.profiles}.created_at) as avatar,
|
||||
argMax(is_external, ${TABLE_NAMES.profiles}.created_at) as is_external,
|
||||
argMax(properties, ${TABLE_NAMES.profiles}.created_at) as properties,
|
||||
max(created_at) as created_at
|
||||
FROM ${TABLE_NAMES.profiles}
|
||||
WHERE (id, project_id) IN (${tuples})
|
||||
${withDateFilter ? `AND ${TABLE_NAMES.profiles}.created_at > now() - INTERVAL 2 DAY` : ''}
|
||||
GROUP BY id, project_id`
|
||||
);
|
||||
for (const row of rows) {
|
||||
result.set(`${row.project_id}:${row.id}`, row);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
'Failed to batch fetch profiles from Clickhouse, proceeding without existing data',
|
||||
{ error, chunkSize: chunk.length }
|
||||
);
|
||||
}
|
||||
GROUP BY id, project_id
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1`
|
||||
);
|
||||
logger.debug('Clickhouse fetch result', {
|
||||
found: !!result[0],
|
||||
});
|
||||
return result[0] || null;
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all([
|
||||
fetchGroup(external, false),
|
||||
fetchGroup(nonExternal, true),
|
||||
]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async processBuffer() {
|
||||
try {
|
||||
this.logger.debug('Starting profile buffer processing');
|
||||
const profiles = await this.redis.lrange(
|
||||
const rawProfiles = await this.redis.lrange(
|
||||
this.redisKey,
|
||||
0,
|
||||
this.batchSize - 1
|
||||
);
|
||||
|
||||
if (profiles.length === 0) {
|
||||
if (rawProfiles.length === 0) {
|
||||
this.logger.debug('No profiles to process');
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
|
||||
const parsedProfiles = profiles.map((p) =>
|
||||
getSafeJson<IClickhouseProfile>(p)
|
||||
);
|
||||
const parsedProfiles = rawProfiles
|
||||
.map((p) => getSafeJson<IClickhouseProfile>(p))
|
||||
.filter(Boolean) as IClickhouseProfile[];
|
||||
|
||||
for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {
|
||||
// Merge within batch: collapse multiple updates for the same profile
|
||||
const mergedInBatch = new Map<string, IClickhouseProfile>();
|
||||
for (const profile of parsedProfiles) {
|
||||
const key = `${profile.project_id}:${profile.id}`;
|
||||
mergedInBatch.set(
|
||||
key,
|
||||
this.mergeProfiles(mergedInBatch.get(key) ?? null, profile)
|
||||
);
|
||||
}
|
||||
|
||||
const uniqueProfiles = Array.from(mergedInBatch.values());
|
||||
|
||||
// Check Redis cache for all unique profiles in a single MGET
|
||||
const cacheKeys = uniqueProfiles.map((p) =>
|
||||
this.getProfileCacheKey({ profileId: p.id, projectId: p.project_id })
|
||||
);
|
||||
const cacheResults = await this.redis.mget(...cacheKeys);
|
||||
|
||||
const existingByKey = new Map<string, IClickhouseProfile>();
|
||||
const cacheMisses: IClickhouseProfile[] = [];
|
||||
for (let i = 0; i < uniqueProfiles.length; i++) {
|
||||
const uniqueProfile = uniqueProfiles[i];
|
||||
if (uniqueProfile) {
|
||||
const key = `${uniqueProfile.project_id}:${uniqueProfile.id}`;
|
||||
const cached = cacheResults[i]
|
||||
? getSafeJson<IClickhouseProfile>(cacheResults[i]!)
|
||||
: null;
|
||||
if (cached) {
|
||||
existingByKey.set(key, cached);
|
||||
} else {
|
||||
cacheMisses.push(uniqueProfile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch cache misses from ClickHouse in bounded chunks
|
||||
if (cacheMisses.length > 0) {
|
||||
const clickhouseResults =
|
||||
await this.batchFetchFromClickhouse(cacheMisses);
|
||||
for (const [key, profile] of clickhouseResults) {
|
||||
existingByKey.set(key, profile);
|
||||
}
|
||||
}
|
||||
|
||||
// Final merge: in-batch profile + existing (from cache or ClickHouse)
|
||||
const toInsert: IClickhouseProfile[] = [];
|
||||
const multi = this.redis.multi();
|
||||
|
||||
for (const profile of uniqueProfiles) {
|
||||
const key = `${profile.project_id}:${profile.id}`;
|
||||
const merged = this.mergeProfiles(
|
||||
existingByKey.get(key) ?? null,
|
||||
profile
|
||||
);
|
||||
toInsert.push(merged);
|
||||
multi.set(
|
||||
this.getProfileCacheKey({
|
||||
projectId: profile.project_id,
|
||||
profileId: profile.id,
|
||||
}),
|
||||
JSON.stringify(merged),
|
||||
'EX',
|
||||
this.ttlInSeconds
|
||||
);
|
||||
}
|
||||
|
||||
for (const chunk of this.chunks(toInsert, this.chunkSize)) {
|
||||
await ch.insert({
|
||||
table: TABLE_NAMES.profiles,
|
||||
values: chunk,
|
||||
@@ -302,22 +284,21 @@ export class ProfileBuffer extends BaseBuffer {
|
||||
});
|
||||
}
|
||||
|
||||
// Only remove profiles after successful insert and update counter
|
||||
await this.redis
|
||||
.multi()
|
||||
.ltrim(this.redisKey, profiles.length, -1)
|
||||
.decrby(this.bufferCounterKey, profiles.length)
|
||||
.exec();
|
||||
multi
|
||||
.ltrim(this.redisKey, rawProfiles.length, -1)
|
||||
.decrby(this.bufferCounterKey, rawProfiles.length);
|
||||
await multi.exec();
|
||||
|
||||
this.logger.debug('Successfully completed profile processing', {
|
||||
totalProfiles: profiles.length,
|
||||
totalProfiles: rawProfiles.length,
|
||||
uniqueProfiles: uniqueProfiles.length,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to process buffer', { error });
|
||||
}
|
||||
}
|
||||
|
||||
async getBufferSize() {
|
||||
getBufferSize() {
|
||||
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
|
||||
}
|
||||
}
|
||||
|
||||
122
packages/db/src/buffers/session-buffer.test.ts
Normal file
122
packages/db/src/buffers/session-buffer.test.ts
Normal file
@@ -0,0 +1,122 @@
|
||||
import { getRedisCache } from '@openpanel/redis';
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { ch } from '../clickhouse/client';
|
||||
|
||||
vi.mock('../clickhouse/client', () => ({
|
||||
ch: {
|
||||
insert: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
TABLE_NAMES: {
|
||||
sessions: 'sessions',
|
||||
},
|
||||
}));
|
||||
|
||||
import { SessionBuffer } from './session-buffer';
|
||||
import type { IClickhouseEvent } from '../services/event.service';
|
||||
|
||||
const redis = getRedisCache();
|
||||
|
||||
function makeEvent(overrides: Partial<IClickhouseEvent>): IClickhouseEvent {
|
||||
return {
|
||||
id: 'event-1',
|
||||
project_id: 'project-1',
|
||||
profile_id: 'profile-1',
|
||||
device_id: 'device-1',
|
||||
session_id: 'session-1',
|
||||
name: 'screen_view',
|
||||
path: '/home',
|
||||
origin: '',
|
||||
referrer: '',
|
||||
referrer_name: '',
|
||||
referrer_type: '',
|
||||
duration: 0,
|
||||
properties: {},
|
||||
created_at: new Date().toISOString(),
|
||||
groups: [],
|
||||
...overrides,
|
||||
} as IClickhouseEvent;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
const keys = [
|
||||
...await redis.keys('session*'),
|
||||
...await redis.keys('lock:session'),
|
||||
];
|
||||
if (keys.length > 0) await redis.del(...keys);
|
||||
vi.mocked(ch.insert).mockResolvedValue(undefined as any);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
try {
|
||||
await redis.quit();
|
||||
} catch {}
|
||||
});
|
||||
|
||||
describe('SessionBuffer', () => {
|
||||
let sessionBuffer: SessionBuffer;
|
||||
|
||||
beforeEach(() => {
|
||||
sessionBuffer = new SessionBuffer();
|
||||
});
|
||||
|
||||
it('adds a new session to the buffer', async () => {
|
||||
const sizeBefore = await sessionBuffer.getBufferSize();
|
||||
await sessionBuffer.add(makeEvent({}));
|
||||
const sizeAfter = await sessionBuffer.getBufferSize();
|
||||
|
||||
expect(sizeAfter).toBe(sizeBefore + 1);
|
||||
});
|
||||
|
||||
it('skips session_start and session_end events', async () => {
|
||||
const sizeBefore = await sessionBuffer.getBufferSize();
|
||||
await sessionBuffer.add(makeEvent({ name: 'session_start' }));
|
||||
await sessionBuffer.add(makeEvent({ name: 'session_end' }));
|
||||
const sizeAfter = await sessionBuffer.getBufferSize();
|
||||
|
||||
expect(sizeAfter).toBe(sizeBefore);
|
||||
});
|
||||
|
||||
it('updates existing session on subsequent events', async () => {
|
||||
const t0 = Date.now();
|
||||
await sessionBuffer.add(makeEvent({ created_at: new Date(t0).toISOString() }));
|
||||
|
||||
// Second event updates the same session — emits old (sign=-1) + new (sign=1)
|
||||
const sizeBefore = await sessionBuffer.getBufferSize();
|
||||
await sessionBuffer.add(makeEvent({ created_at: new Date(t0 + 5000).toISOString() }));
|
||||
const sizeAfter = await sessionBuffer.getBufferSize();
|
||||
|
||||
expect(sizeAfter).toBe(sizeBefore + 2);
|
||||
});
|
||||
|
||||
it('processes buffer and inserts sessions into ClickHouse', async () => {
|
||||
await sessionBuffer.add(makeEvent({}));
|
||||
|
||||
const insertSpy = vi
|
||||
.spyOn(ch, 'insert')
|
||||
.mockResolvedValueOnce(undefined as any);
|
||||
|
||||
await sessionBuffer.processBuffer();
|
||||
|
||||
expect(insertSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
|
||||
);
|
||||
expect(await sessionBuffer.getBufferSize()).toBe(0);
|
||||
|
||||
insertSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('retains sessions in queue when ClickHouse insert fails', async () => {
|
||||
await sessionBuffer.add(makeEvent({}));
|
||||
|
||||
const insertSpy = vi
|
||||
.spyOn(ch, 'insert')
|
||||
.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
|
||||
|
||||
await sessionBuffer.processBuffer();
|
||||
|
||||
// Sessions must still be in the queue — not lost
|
||||
expect(await sessionBuffer.getBufferSize()).toBe(1);
|
||||
|
||||
insertSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -9,10 +9,10 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@react-email/components": "^0.5.6",
|
||||
"@react-email/components": "^1.0.10",
|
||||
"react": "catalog:",
|
||||
"react-dom": "catalog:",
|
||||
"resend": "^4.0.1",
|
||||
"resend": "^4.8.0",
|
||||
"responsive-react-email": "^0.0.5",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
|
||||
@@ -7,15 +7,15 @@
|
||||
"codegen": "jiti scripts/download.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@maxmind/geoip2-node": "^6.1.0",
|
||||
"lru-cache": "^11.2.2"
|
||||
"@maxmind/geoip2-node": "^6.3.4",
|
||||
"lru-cache": "^11.2.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"fast-extract": "^1.4.3",
|
||||
"jiti": "^2.4.1",
|
||||
"tar": "^7.4.3",
|
||||
"fast-extract": "^1.14.2",
|
||||
"jiti": "^2.6.1",
|
||||
"tar": "^7.5.13",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,18 +18,18 @@
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"csv-parse": "^6.1.0",
|
||||
"ramda": "^0.29.1",
|
||||
"csv-parse": "^6.2.1",
|
||||
"ramda": "^0.32.0",
|
||||
"uuid": "^9.0.1",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/logger": "workspace:*",
|
||||
"@types/node": "^20.0.0",
|
||||
"@types/node": "^20.19.37",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/uuid": "^9.0.7",
|
||||
"bullmq": "^5.8.7",
|
||||
"typescript": "^5.0.0",
|
||||
"vitest": "^1.0.0"
|
||||
"@types/uuid": "^11.0.0",
|
||||
"bullmq": "^5.71.1",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^1.6.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@slack/bolt": "^3.18.0",
|
||||
"@slack/oauth": "^3.0.0"
|
||||
"@slack/bolt": "^3.22.0",
|
||||
"@slack/oauth": "^3.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
|
||||
@@ -11,11 +11,11 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@babel/parser": "^7.26.0"
|
||||
"@babel/parser": "^7.29.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"typescript": "catalog:",
|
||||
"vitest": "^2.1.8"
|
||||
"vitest": "^2.1.9"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,13 +7,13 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"winston": "^3.14.2"
|
||||
"@hyperdx/node-opentelemetry": "^0.10.3",
|
||||
"winston": "^3.19.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"date-fns": "^3.3.1",
|
||||
"prisma": "^5.1.1",
|
||||
"date-fns": "^3.6.0",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,16 +11,16 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@polar-sh/sdk": "^0.35.4"
|
||||
"@polar-sh/sdk": "^0.46.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/inquirer": "^9.0.7",
|
||||
"@types/inquirer": "^9.0.9",
|
||||
"@types/inquirer-autocomplete-prompt": "^3.0.3",
|
||||
"@types/node": "catalog:",
|
||||
"@types/react": "catalog:",
|
||||
"inquirer": "^9.3.5",
|
||||
"inquirer": "^9.3.8",
|
||||
"inquirer-autocomplete-prompt": "^3.0.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/logger": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"bullmq": "^5.63.0",
|
||||
"bullmq": "^5.71.1",
|
||||
"groupmq": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -8,7 +8,7 @@ import { createLogger } from '@openpanel/logger';
|
||||
import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
|
||||
import { Queue } from 'bullmq';
|
||||
import { Queue as GroupQueue } from 'groupmq';
|
||||
import type { ITrackPayload } from '../../validation';
|
||||
import type { ILogPayload, ITrackPayload } from '../../validation';
|
||||
|
||||
export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(
|
||||
process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',
|
||||
@@ -138,6 +138,10 @@ export type CronQueuePayloadFlushGroups = {
|
||||
type: 'flushGroups';
|
||||
payload: undefined;
|
||||
};
|
||||
export type CronQueuePayloadFlushLogs = {
|
||||
type: 'flushLogs';
|
||||
payload: undefined;
|
||||
};
|
||||
export type CronQueuePayload =
|
||||
| CronQueuePayloadSalt
|
||||
| CronQueuePayloadFlushEvents
|
||||
@@ -146,6 +150,7 @@ export type CronQueuePayload =
|
||||
| CronQueuePayloadFlushProfileBackfill
|
||||
| CronQueuePayloadFlushReplay
|
||||
| CronQueuePayloadFlushGroups
|
||||
| CronQueuePayloadFlushLogs
|
||||
| CronQueuePayloadPing
|
||||
| CronQueuePayloadProject
|
||||
| CronQueuePayloadInsightsDaily
|
||||
@@ -297,3 +302,50 @@ export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {
|
||||
removeOnFail: 100,
|
||||
},
|
||||
});
|
||||
|
||||
export type LogsQueuePayload = {
|
||||
type: 'incomingLog';
|
||||
payload: {
|
||||
projectId: string;
|
||||
log: ILogPayload & {
|
||||
timestamp: string;
|
||||
};
|
||||
uaInfo:
|
||||
| {
|
||||
readonly isServer: true;
|
||||
readonly device: 'server';
|
||||
readonly os: '';
|
||||
readonly osVersion: '';
|
||||
readonly browser: '';
|
||||
readonly browserVersion: '';
|
||||
readonly brand: '';
|
||||
readonly model: '';
|
||||
}
|
||||
| {
|
||||
readonly os: string | undefined;
|
||||
readonly osVersion: string | undefined;
|
||||
readonly browser: string | undefined;
|
||||
readonly browserVersion: string | undefined;
|
||||
readonly device: string;
|
||||
readonly brand: string | undefined;
|
||||
readonly model: string | undefined;
|
||||
readonly isServer: false;
|
||||
};
|
||||
geo: {
|
||||
country: string | undefined;
|
||||
city: string | undefined;
|
||||
region: string | undefined;
|
||||
};
|
||||
headers: Record<string, string | undefined>;
|
||||
deviceId: string;
|
||||
sessionId: string;
|
||||
};
|
||||
};
|
||||
|
||||
export const logsQueue = new Queue<LogsQueuePayload>(getQueueName('logs'), {
|
||||
connection: getRedisQueue(),
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 1000,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -8,7 +8,10 @@ describe('cachable', () => {
|
||||
beforeEach(async () => {
|
||||
redis = getRedisCache();
|
||||
// Clear any existing cache data for clean tests
|
||||
const keys = await redis.keys('cachable:*');
|
||||
const keys = [
|
||||
...await redis.keys('cachable:*'),
|
||||
...await redis.keys('test-key*'),
|
||||
];
|
||||
if (keys.length > 0) {
|
||||
await redis.del(...keys);
|
||||
}
|
||||
@@ -16,7 +19,10 @@ describe('cachable', () => {
|
||||
|
||||
afterEach(async () => {
|
||||
// Clean up after each test
|
||||
const keys = await redis.keys('cachable:*');
|
||||
const keys = [
|
||||
...await redis.keys('cachable:*'),
|
||||
...await redis.keys('test-key*'),
|
||||
];
|
||||
if (keys.length > 0) {
|
||||
await redis.del(...keys);
|
||||
}
|
||||
|
||||
@@ -8,14 +8,14 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/json": "workspace:*",
|
||||
"ioredis": "5.8.2",
|
||||
"lru-cache": "^11.2.2"
|
||||
"ioredis": "5.10.1",
|
||||
"lru-cache": "^11.2.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
"react": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "catalog:",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"prisma": "^5.1.1",
|
||||
"@types/react": "catalog:",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"@openpanel/web": "workspace:1.3.0-local"
|
||||
},
|
||||
"devDependencies": {
|
||||
"astro": "^5.7.7"
|
||||
"astro": "^5.18.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"astro": "^4.0.0 || ^5.0.0"
|
||||
|
||||
@@ -10,17 +10,17 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/sdk": "workspace:1.3.0-local",
|
||||
"@openpanel/common": "workspace:*"
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/sdk": "workspace:1.3.0-local"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"express": "^4.17.0 || ^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/express": "^5.0.3",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/request-ip": "^0.0.41",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/react": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,12 +31,12 @@
|
||||
"nuxt": "^3.0.0 || ^4.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nuxt/kit": "^3.0.0",
|
||||
"@nuxt/kit": "^3.21.2",
|
||||
"@nuxt/module-builder": "^1.0.2",
|
||||
"@nuxt/types": "^2.18.1",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@vue/runtime-core": "^3.5.25",
|
||||
"@vue/runtime-core": "^3.5.31",
|
||||
"typescript": "catalog:",
|
||||
"unbuild": "^3.6.1"
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -9,12 +9,11 @@
|
||||
"build": "rm -rf dist && tsup",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
IGroupPayload as GroupPayload,
|
||||
IIdentifyPayload as IdentifyPayload,
|
||||
IIncrementPayload as IncrementPayload,
|
||||
ISeverityText,
|
||||
ITrackHandlerPayload as TrackHandlerPayload,
|
||||
ITrackPayload as TrackPayload,
|
||||
} from '@openpanel/validation';
|
||||
@@ -19,6 +20,7 @@ export type {
|
||||
GroupPayload,
|
||||
IdentifyPayload,
|
||||
IncrementPayload,
|
||||
ISeverityText,
|
||||
TrackHandlerPayload,
|
||||
TrackPayload,
|
||||
};
|
||||
@@ -29,6 +31,33 @@ export interface TrackProperties {
|
||||
groups?: string[];
|
||||
}
|
||||
|
||||
export interface LogProperties {
|
||||
/** Logger name (e.g. "com.example.MyActivity") */
|
||||
loggerName?: string;
|
||||
traceId?: string;
|
||||
spanId?: string;
|
||||
traceFlags?: number;
|
||||
/** Log-level key-value attributes */
|
||||
attributes?: Record<string, string>;
|
||||
/** Resource/device attributes */
|
||||
resource?: Record<string, string>;
|
||||
/** ISO 8601 timestamp; defaults to now */
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
interface LogPayloadForQueue {
|
||||
body: string;
|
||||
severity: ISeverityText;
|
||||
loggerName?: string;
|
||||
traceId?: string;
|
||||
spanId?: string;
|
||||
traceFlags?: number;
|
||||
attributes?: Record<string, string>;
|
||||
resource?: Record<string, string>;
|
||||
timestamp: string;
|
||||
profileId?: string;
|
||||
}
|
||||
|
||||
export type UpsertGroupPayload = GroupPayload;
|
||||
|
||||
export interface OpenPanelOptions {
|
||||
@@ -57,6 +86,10 @@ export class OpenPanel {
|
||||
sessionId?: string;
|
||||
global?: Record<string, unknown>;
|
||||
queue: TrackHandlerPayload[] = [];
|
||||
private logQueue: LogPayloadForQueue[] = [];
|
||||
private logFlushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private logFlushIntervalMs = 2000;
|
||||
private logFlushMaxSize = 50;
|
||||
|
||||
constructor(options: OpenPanelOptions) {
|
||||
this.options = options;
|
||||
@@ -327,6 +360,67 @@ export class OpenPanel {
|
||||
this.queue = remaining;
|
||||
}
|
||||
|
||||
captureLog(
|
||||
severity: ISeverityText,
|
||||
body: string,
|
||||
properties?: LogProperties,
|
||||
) {
|
||||
if (this.options.disabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: LogPayloadForQueue = {
|
||||
body,
|
||||
severity,
|
||||
timestamp: properties?.timestamp ?? new Date().toISOString(),
|
||||
...(this.profileId ? { profileId: this.profileId } : {}),
|
||||
...(properties?.loggerName ? { loggerName: properties.loggerName } : {}),
|
||||
...(properties?.traceId ? { traceId: properties.traceId } : {}),
|
||||
...(properties?.spanId ? { spanId: properties.spanId } : {}),
|
||||
...(properties?.traceFlags !== undefined
|
||||
? { traceFlags: properties.traceFlags }
|
||||
: {}),
|
||||
...(properties?.attributes ? { attributes: properties.attributes } : {}),
|
||||
...(properties?.resource ? { resource: properties.resource } : {}),
|
||||
};
|
||||
|
||||
this.logQueue.push(entry);
|
||||
|
||||
if (this.logQueue.length >= this.logFlushMaxSize) {
|
||||
this.flushLogs();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.logFlushTimer) {
|
||||
this.logFlushTimer = setTimeout(() => {
|
||||
this.logFlushTimer = null;
|
||||
this.flushLogs();
|
||||
}, this.logFlushIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private async flushLogs() {
|
||||
if (this.logFlushTimer) {
|
||||
clearTimeout(this.logFlushTimer);
|
||||
this.logFlushTimer = null;
|
||||
}
|
||||
|
||||
if (this.logQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const batch = this.logQueue;
|
||||
this.logQueue = [];
|
||||
|
||||
try {
|
||||
await this.api.fetch('/logs', { logs: batch });
|
||||
} catch (error) {
|
||||
this.log('Failed to flush logs', error);
|
||||
// Re-queue on failure
|
||||
this.logQueue = batch.concat(this.logQueue);
|
||||
}
|
||||
}
|
||||
|
||||
log(...args: any[]) {
|
||||
if (this.options.debug) {
|
||||
console.log('[OpenPanel.dev]', ...args);
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export * from './src/root';
|
||||
export * from './src/trpc';
|
||||
export { getProjectAccess } from './src/access';
|
||||
export type { IServiceLog } from './src/routers/log';
|
||||
|
||||
@@ -16,17 +16,17 @@
|
||||
"@openpanel/integrations": "workspace:^",
|
||||
"@openpanel/js-runtime": "workspace:*",
|
||||
"@openpanel/payments": "workspace:^",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@trpc-limiter/redis": "^0.0.2",
|
||||
"@trpc/client": "^11.6.0",
|
||||
"@trpc/server": "^11.6.0",
|
||||
"date-fns": "^3.3.1",
|
||||
"mathjs": "^12.3.2",
|
||||
"@trpc/client": "^11.16.0",
|
||||
"@trpc/server": "^11.16.0",
|
||||
"date-fns": "^3.6.0",
|
||||
"mathjs": "^15.1.1",
|
||||
"prisma-error-enum": "^0.1.3",
|
||||
"ramda": "^0.29.1",
|
||||
"short-unique-id": "^5.0.3",
|
||||
"ramda": "^0.32.0",
|
||||
"short-unique-id": "^5.3.2",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"uuid": "^9.0.1",
|
||||
@@ -35,9 +35,9 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import { eventRouter } from './routers/event';
|
||||
import { groupRouter } from './routers/group';
|
||||
import { gscRouter } from './routers/gsc';
|
||||
import { importRouter } from './routers/import';
|
||||
import { logRouter } from './routers/log';
|
||||
import { insightRouter } from './routers/insight';
|
||||
import { integrationRouter } from './routers/integration';
|
||||
import { notificationRouter } from './routers/notification';
|
||||
@@ -57,6 +58,7 @@ export const appRouter = createTRPCRouter({
|
||||
email: emailRouter,
|
||||
gsc: gscRouter,
|
||||
group: groupRouter,
|
||||
log: logRouter,
|
||||
});
|
||||
|
||||
// export type definition of API
|
||||
|
||||
212
packages/trpc/src/routers/log.ts
Normal file
212
packages/trpc/src/routers/log.ts
Normal file
@@ -0,0 +1,212 @@
|
||||
import { chQuery, convertClickhouseDateToJs } from '@openpanel/db';
|
||||
import { zSeverityText } from '@openpanel/validation';
|
||||
import sqlstring from 'sqlstring';
|
||||
import { z } from 'zod';
|
||||
import { createTRPCRouter, protectedProcedure } from '../trpc';
|
||||
|
||||
export interface IServiceLog {
|
||||
id: string;
|
||||
projectId: string;
|
||||
deviceId: string;
|
||||
profileId: string;
|
||||
sessionId: string;
|
||||
timestamp: Date;
|
||||
severityNumber: number;
|
||||
severityText: string;
|
||||
body: string;
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
traceFlags: number;
|
||||
loggerName: string;
|
||||
attributes: Record<string, string>;
|
||||
resource: Record<string, string>;
|
||||
sdkName: string;
|
||||
sdkVersion: string;
|
||||
country: string;
|
||||
city: string;
|
||||
region: string;
|
||||
os: string;
|
||||
osVersion: string;
|
||||
browser: string;
|
||||
browserVersion: string;
|
||||
device: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
interface IClickhouseLog {
|
||||
id: string;
|
||||
project_id: string;
|
||||
device_id: string;
|
||||
profile_id: string;
|
||||
session_id: string;
|
||||
timestamp: string;
|
||||
severity_number: number;
|
||||
severity_text: string;
|
||||
body: string;
|
||||
trace_id: string;
|
||||
span_id: string;
|
||||
trace_flags: number;
|
||||
logger_name: string;
|
||||
attributes: Record<string, string>;
|
||||
resource: Record<string, string>;
|
||||
sdk_name: string;
|
||||
sdk_version: string;
|
||||
country: string;
|
||||
city: string;
|
||||
region: string;
|
||||
os: string;
|
||||
os_version: string;
|
||||
browser: string;
|
||||
browser_version: string;
|
||||
device: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
function toServiceLog(row: IClickhouseLog): IServiceLog {
|
||||
return {
|
||||
id: row.id,
|
||||
projectId: row.project_id,
|
||||
deviceId: row.device_id,
|
||||
profileId: row.profile_id,
|
||||
sessionId: row.session_id,
|
||||
timestamp: convertClickhouseDateToJs(row.timestamp),
|
||||
severityNumber: row.severity_number,
|
||||
severityText: row.severity_text,
|
||||
body: row.body,
|
||||
traceId: row.trace_id,
|
||||
spanId: row.span_id,
|
||||
traceFlags: row.trace_flags,
|
||||
loggerName: row.logger_name,
|
||||
attributes: row.attributes,
|
||||
resource: row.resource,
|
||||
sdkName: row.sdk_name,
|
||||
sdkVersion: row.sdk_version,
|
||||
country: row.country,
|
||||
city: row.city,
|
||||
region: row.region,
|
||||
os: row.os,
|
||||
osVersion: row.os_version,
|
||||
browser: row.browser,
|
||||
browserVersion: row.browser_version,
|
||||
device: row.device,
|
||||
brand: row.brand,
|
||||
model: row.model,
|
||||
};
|
||||
}
|
||||
|
||||
export const logRouter = createTRPCRouter({
|
||||
list: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
projectId: z.string(),
|
||||
cursor: z.string().nullish(),
|
||||
severity: z.array(zSeverityText).optional(),
|
||||
search: z.string().optional(),
|
||||
loggerName: z.string().optional(),
|
||||
startDate: z.date().optional(),
|
||||
endDate: z.date().optional(),
|
||||
take: z.number().default(50),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input }) => {
|
||||
const { projectId, cursor, severity, search, loggerName, startDate, endDate, take } = input;
|
||||
|
||||
const conditions: string[] = [
|
||||
`project_id = ${sqlstring.escape(projectId)}`,
|
||||
];
|
||||
|
||||
if (cursor) {
|
||||
conditions.push(`timestamp < ${sqlstring.escape(cursor)}`);
|
||||
}
|
||||
|
||||
if (severity && severity.length > 0) {
|
||||
const escaped = severity.map((s) => sqlstring.escape(s)).join(', ');
|
||||
conditions.push(`severity_text IN (${escaped})`);
|
||||
}
|
||||
|
||||
if (search) {
|
||||
conditions.push(`body ILIKE ${sqlstring.escape(`%${search}%`)}`);
|
||||
}
|
||||
|
||||
if (loggerName) {
|
||||
conditions.push(`logger_name = ${sqlstring.escape(loggerName)}`);
|
||||
}
|
||||
|
||||
if (startDate) {
|
||||
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
|
||||
}
|
||||
|
||||
if (endDate) {
|
||||
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
|
||||
}
|
||||
|
||||
const where = conditions.join(' AND ');
|
||||
|
||||
const rows = await chQuery<IClickhouseLog>(
|
||||
`SELECT
|
||||
id, project_id, device_id, profile_id, session_id,
|
||||
timestamp, severity_number, severity_text, body,
|
||||
trace_id, span_id, trace_flags, logger_name,
|
||||
attributes, resource,
|
||||
sdk_name, sdk_version,
|
||||
country, city, region, os, os_version,
|
||||
browser, browser_version, device, brand, model
|
||||
FROM logs
|
||||
WHERE ${where}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ${take + 1}`,
|
||||
);
|
||||
|
||||
const hasMore = rows.length > take;
|
||||
const data = rows.slice(0, take).map(toServiceLog);
|
||||
const lastItem = data[data.length - 1];
|
||||
|
||||
return {
|
||||
data,
|
||||
meta: {
|
||||
next: hasMore && lastItem ? lastItem.timestamp.toISOString() : null,
|
||||
},
|
||||
};
|
||||
}),
|
||||
|
||||
severityCounts: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
projectId: z.string(),
|
||||
startDate: z.date().optional(),
|
||||
endDate: z.date().optional(),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input }) => {
|
||||
const { projectId, startDate, endDate } = input;
|
||||
|
||||
const conditions: string[] = [
|
||||
`project_id = ${sqlstring.escape(projectId)}`,
|
||||
];
|
||||
|
||||
if (startDate) {
|
||||
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
|
||||
}
|
||||
|
||||
if (endDate) {
|
||||
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
|
||||
}
|
||||
|
||||
const where = conditions.join(' AND ');
|
||||
|
||||
const rows = await chQuery<{ severity_text: string; count: number }>(
|
||||
`SELECT severity_text, count() AS count
|
||||
FROM logs
|
||||
WHERE ${where}
|
||||
GROUP BY severity_text
|
||||
ORDER BY count DESC`,
|
||||
);
|
||||
|
||||
return rows.reduce<Record<string, number>>((acc, row) => {
|
||||
acc[row.severity_text] = row.count;
|
||||
return acc;
|
||||
}, {});
|
||||
}),
|
||||
});
|
||||
@@ -13,7 +13,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,3 +626,4 @@ export type ICreateImport = z.infer<typeof zCreateImport>;
|
||||
export * from './types.insights';
|
||||
export * from './track.validation';
|
||||
export * from './event-blocklist';
|
||||
export * from './log.validation';
|
||||
|
||||
60
packages/validation/src/log.validation.ts
Normal file
60
packages/validation/src/log.validation.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* OTel severity number mapping (subset):
|
||||
* TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21
|
||||
*/
|
||||
export const SEVERITY_TEXT_TO_NUMBER: Record<string, number> = {
|
||||
trace: 1,
|
||||
debug: 5,
|
||||
info: 9,
|
||||
warn: 13,
|
||||
warning: 13,
|
||||
error: 17,
|
||||
fatal: 21,
|
||||
critical: 21,
|
||||
};
|
||||
|
||||
export const zSeverityText = z.enum([
|
||||
'trace',
|
||||
'debug',
|
||||
'info',
|
||||
'warn',
|
||||
'warning',
|
||||
'error',
|
||||
'fatal',
|
||||
'critical',
|
||||
]);
|
||||
|
||||
export type ISeverityText = z.infer<typeof zSeverityText>;
|
||||
|
||||
export const zLogPayload = z.object({
|
||||
/** Log message / body */
|
||||
body: z.string().min(1),
|
||||
/** Severity level as text */
|
||||
severity: zSeverityText.default('info'),
|
||||
/** Optional override for the numeric OTel severity (1-24) */
|
||||
severityNumber: z.number().int().min(1).max(24).optional(),
|
||||
/** ISO 8601 timestamp; defaults to server receive time if omitted */
|
||||
timestamp: z.string().datetime({ offset: true }).optional(),
|
||||
/** Logger name (e.g. "com.example.MyActivity") */
|
||||
loggerName: z.string().optional(),
|
||||
/** W3C trace context */
|
||||
traceId: z.string().optional(),
|
||||
spanId: z.string().optional(),
|
||||
traceFlags: z.number().int().min(0).optional(),
|
||||
/** Log-level key-value attributes */
|
||||
attributes: z.record(z.string(), z.string()).optional(),
|
||||
/** Resource/device attributes (app version, runtime, etc.) */
|
||||
resource: z.record(z.string(), z.string()).optional(),
|
||||
/** Profile/user ID to associate with this log */
|
||||
profileId: z.union([z.string().min(1), z.number()]).optional(),
|
||||
});
|
||||
|
||||
export type ILogPayload = z.infer<typeof zLogPayload>;
|
||||
|
||||
export const zLogBatchPayload = z.object({
|
||||
logs: z.array(zLogPayload).min(1).max(500),
|
||||
});
|
||||
|
||||
export type ILogBatchPayload = z.infer<typeof zLogBatchPayload>;
|
||||
42
patches/buffer-equal-constant-time.patch
Normal file
42
patches/buffer-equal-constant-time.patch
Normal file
@@ -0,0 +1,42 @@
|
||||
diff --git a/.npmignore b/.npmignore
|
||||
deleted file mode 100644
|
||||
index 34e4f5c298de294fa5c1c1769b6489eb047bde9a..0000000000000000000000000000000000000000
|
||||
diff --git a/index.js b/index.js
|
||||
index 5462c1f830bdbe79bf2b1fcfd811cd9799b4dd11..689421d49e83d168981a0c7d1fef59a0b0e56963 100644
|
||||
--- a/index.js
|
||||
+++ b/index.js
|
||||
@@ -3,6 +3,9 @@
|
||||
var Buffer = require('buffer').Buffer; // browserify
|
||||
var SlowBuffer = require('buffer').SlowBuffer;
|
||||
|
||||
+// Handle Node.js v25+ where SlowBuffer was removed
|
||||
+var hasSlowBuffer = !!SlowBuffer;
|
||||
+
|
||||
module.exports = bufferEq;
|
||||
|
||||
function bufferEq(a, b) {
|
||||
@@ -28,14 +31,18 @@ function bufferEq(a, b) {
|
||||
}
|
||||
|
||||
bufferEq.install = function() {
|
||||
- Buffer.prototype.equal = SlowBuffer.prototype.equal = function equal(that) {
|
||||
- return bufferEq(this, that);
|
||||
- };
|
||||
+ Buffer.prototype.equal = function equal(that) {
|
||||
+ return bufferEq(this, that);
|
||||
+ };
|
||||
+ if (hasSlowBuffer) {
|
||||
+ SlowBuffer.prototype.equal = Buffer.prototype.equal;
|
||||
+ }
|
||||
};
|
||||
|
||||
var origBufEqual = Buffer.prototype.equal;
|
||||
-var origSlowBufEqual = SlowBuffer.prototype.equal;
|
||||
bufferEq.restore = function() {
|
||||
- Buffer.prototype.equal = origBufEqual;
|
||||
- SlowBuffer.prototype.equal = origSlowBufEqual;
|
||||
+ Buffer.prototype.equal = origBufEqual;
|
||||
+ if (hasSlowBuffer && SlowBuffer.prototype) {
|
||||
+ delete SlowBuffer.prototype.equal;
|
||||
+ }
|
||||
};
|
||||
28713
pnpm-lock.yaml
generated
28713
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -7,10 +7,10 @@ packages:
|
||||
# Define a catalog of version ranges.
|
||||
catalog:
|
||||
zod: ^3.24.2
|
||||
react: ^19.2.3
|
||||
"@types/react": ^19.2.3
|
||||
"react-dom": ^19.2.3
|
||||
react: ^19.2.4
|
||||
"@types/react": ^19.2.14
|
||||
"react-dom": ^19.2.4
|
||||
"@types/react-dom": ^19.2.3
|
||||
"@types/node": ^24.7.1
|
||||
"@types/node": ^25.5.0
|
||||
typescript: ^5.9.3
|
||||
groupmq: 2.0.0-next.1
|
||||
|
||||
@@ -23,7 +23,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
op-db:
|
||||
image: postgres:14-alpine
|
||||
image: postgres:18.3-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-db-data:/var/lib/postgresql/data
|
||||
@@ -45,7 +45,7 @@ services:
|
||||
# - 5432:5432
|
||||
|
||||
op-kv:
|
||||
image: redis:7.2.5-alpine
|
||||
image: redis:8.6.2-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-kv-data:/data
|
||||
@@ -65,7 +65,7 @@ services:
|
||||
# - 6379:6379
|
||||
|
||||
op-ch:
|
||||
image: clickhouse/clickhouse-server:25.10.2.65
|
||||
image: clickhouse/clickhouse-server:26.3.2.3
|
||||
restart: always
|
||||
volumes:
|
||||
- op-ch-data:/var/lib/clickhouse
|
||||
|
||||
@@ -9,12 +9,12 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"arg": "^5.0.2",
|
||||
"semver": "^7.5.4"
|
||||
"semver": "^7.7.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/semver": "^7.5.4",
|
||||
"@types/semver": "^7.7.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"files": ["base.json", "sdk.json", "tsup.config.json"],
|
||||
"files": [
|
||||
"base.json",
|
||||
"sdk.json",
|
||||
"tsup.config.json"
|
||||
],
|
||||
"devDependencies": {
|
||||
"tsup": "^7.2.0"
|
||||
"tsup": "^8.5.1"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user