Compare commits
3 Commits
a1ce71ffb6
...
be64494aa9
| Author | SHA1 | Date | |
|---|---|---|---|
|
be64494aa9
|
|||
|
eefbeac7f8
|
|||
|
0672857974
|
55
.gitea/workflows/docker-build-api.yml
Normal file
55
.gitea/workflows/docker-build-api.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
name: Build and Push API
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.zias.be
|
||||
OWNER: zias
|
||||
|
||||
jobs:
|
||||
build-api:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ env.OWNER }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=sha-,format=short
|
||||
type=semver,pattern={{version}}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/api/Dockerfile
|
||||
target: runner
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-api:buildcache
|
||||
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-api:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
|
||||
build-args: |
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres
|
||||
53
.gitea/workflows/docker-build-dashboard.yml
Normal file
53
.gitea/workflows/docker-build-dashboard.yml
Normal file
@@ -0,0 +1,53 @@
|
||||
name: Build and Push Dashboard
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.zias.be
|
||||
OWNER: zias
|
||||
|
||||
jobs:
|
||||
build-dashboard:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ env.OWNER }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=sha-,format=short
|
||||
type=semver,pattern={{version}}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/start/Dockerfile
|
||||
target: runner
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-dashboard:buildcache
|
||||
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-dashboard:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
|
||||
55
.gitea/workflows/docker-build-worker.yml
Normal file
55
.gitea/workflows/docker-build-worker.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
name: Build and Push Worker
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
REGISTRY: git.zias.be
|
||||
OWNER: zias
|
||||
|
||||
jobs:
|
||||
build-worker:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ env.OWNER }}
|
||||
password: ${{ secrets.REGISTRY_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker
|
||||
tags: |
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=sha,prefix=sha-,format=short
|
||||
type=semver,pattern={{version}}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/worker/Dockerfile
|
||||
target: runner
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: false
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.OWNER }}/openpanel-worker:buildcache
|
||||
cache-to: ${{ github.event_name != 'pull_request' && format('type=registry,ref={0}/{1}/openpanel-worker:buildcache,mode=max,image-manifest=true,oci-mediatypes=true', env.REGISTRY, env.OWNER) || '' }}
|
||||
build-args: |
|
||||
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres
|
||||
@@ -10,14 +10,14 @@
|
||||
"dependencies": {
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/db": "workspace:*",
|
||||
"chalk": "^5.3.0",
|
||||
"chalk": "^5.6.2",
|
||||
"fuzzy": "^0.1.3",
|
||||
"inquirer": "^9.3.5",
|
||||
"inquirer": "^9.3.8",
|
||||
"inquirer-autocomplete-prompt": "^3.0.1",
|
||||
"jiti": "^2.4.2"
|
||||
"jiti": "^2.6.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/inquirer": "^9.0.7",
|
||||
"@types/inquirer": "^9.0.9",
|
||||
"@types/inquirer-autocomplete-prompt": "^3.0.3",
|
||||
"@types/node": "catalog:",
|
||||
"typescript": "catalog:"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG NODE_VERSION=22.20.0
|
||||
ARG NODE_VERSION=22.22.2
|
||||
|
||||
FROM node:${NODE_VERSION}-slim AS base
|
||||
|
||||
|
||||
@@ -12,11 +12,11 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^1.2.10",
|
||||
"@ai-sdk/openai": "^1.3.12",
|
||||
"@fastify/compress": "^8.1.0",
|
||||
"@ai-sdk/anthropic": "^1.2.12",
|
||||
"@ai-sdk/openai": "^1.3.24",
|
||||
"@fastify/compress": "^8.3.1",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/cors": "^11.1.0",
|
||||
"@fastify/cors": "^11.2.0",
|
||||
"@fastify/rate-limit": "^10.3.0",
|
||||
"@fastify/websocket": "^11.2.0",
|
||||
"@node-rs/argon2": "^2.0.2",
|
||||
@@ -33,36 +33,36 @@
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/trpc": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@trpc/server": "^11.6.0",
|
||||
"ai": "^4.2.10",
|
||||
"@trpc/server": "^11.16.0",
|
||||
"ai": "^4.3.19",
|
||||
"fast-json-stable-hash": "^1.0.3",
|
||||
"fastify": "^5.6.1",
|
||||
"fastify": "^5.8.4",
|
||||
"fastify-metrics": "^12.1.0",
|
||||
"fastify-raw-body": "^5.0.0",
|
||||
"groupmq": "catalog:",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"ramda": "^0.29.1",
|
||||
"sharp": "^0.33.5",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"ramda": "^0.32.0",
|
||||
"sharp": "^0.34.5",
|
||||
"source-map-support": "^0.5.21",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"svix": "^1.24.0",
|
||||
"url-metadata": "^5.4.1",
|
||||
"svix": "^1.89.0",
|
||||
"url-metadata": "^5.4.3",
|
||||
"uuid": "^9.0.1",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@faker-js/faker": "^9.0.1",
|
||||
"@faker-js/faker": "^9.9.0",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
"@types/jsonwebtoken": "^9.0.9",
|
||||
"@types/ramda": "^0.30.2",
|
||||
"@types/jsonwebtoken": "^9.0.10",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/source-map-support": "^0.5.10",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"@types/ws": "^8.5.14",
|
||||
"js-yaml": "^4.1.0",
|
||||
"tsdown": "0.14.2",
|
||||
"@types/uuid": "^11.0.0",
|
||||
"@types/ws": "^8.18.1",
|
||||
"js-yaml": "^4.1.1",
|
||||
"tsdown": "0.21.7",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
68
apps/api/src/controllers/logs.controller.ts
Normal file
68
apps/api/src/controllers/logs.controller.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { parseUserAgent } from '@openpanel/common/server';
|
||||
import { getSalts } from '@openpanel/db';
|
||||
import { getGeoLocation } from '@openpanel/geo';
|
||||
import { type LogsQueuePayload, logsQueue } from '@openpanel/queue';
|
||||
import { type ILogBatchPayload, zLogBatchPayload } from '@openpanel/validation';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { getDeviceId } from '@/utils/ids';
|
||||
import { getStringHeaders } from './track.controller';
|
||||
|
||||
export async function handler(
|
||||
request: FastifyRequest<{ Body: ILogBatchPayload }>,
|
||||
reply: FastifyReply,
|
||||
) {
|
||||
const projectId = request.client?.projectId;
|
||||
if (!projectId) {
|
||||
return reply.status(400).send({ status: 400, error: 'Missing projectId' });
|
||||
}
|
||||
|
||||
const validationResult = zLogBatchPayload.safeParse(request.body);
|
||||
if (!validationResult.success) {
|
||||
return reply.status(400).send({
|
||||
status: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Validation failed',
|
||||
errors: validationResult.error.errors,
|
||||
});
|
||||
}
|
||||
|
||||
const { logs } = validationResult.data;
|
||||
|
||||
const ip = request.clientIp;
|
||||
const ua = request.headers['user-agent'] ?? 'unknown/1.0';
|
||||
const headers = getStringHeaders(request.headers);
|
||||
const receivedAt = new Date().toISOString();
|
||||
|
||||
const [geo, salts] = await Promise.all([getGeoLocation(ip), getSalts()]);
|
||||
const { deviceId, sessionId } = await getDeviceId({ projectId, ip, ua, salts });
|
||||
const uaInfo = parseUserAgent(ua, undefined);
|
||||
|
||||
const jobs: LogsQueuePayload[] = logs.map((log) => ({
|
||||
type: 'incomingLog' as const,
|
||||
payload: {
|
||||
projectId,
|
||||
log: {
|
||||
...log,
|
||||
timestamp: log.timestamp ?? receivedAt,
|
||||
},
|
||||
uaInfo,
|
||||
geo: {
|
||||
country: geo.country,
|
||||
city: geo.city,
|
||||
region: geo.region,
|
||||
},
|
||||
headers,
|
||||
deviceId,
|
||||
sessionId,
|
||||
},
|
||||
}));
|
||||
|
||||
await logsQueue.addBulk(
|
||||
jobs.map((job) => ({
|
||||
name: 'incomingLog',
|
||||
data: job,
|
||||
})),
|
||||
);
|
||||
|
||||
return reply.status(200).send({ ok: true, count: logs.length });
|
||||
}
|
||||
@@ -44,6 +44,7 @@ import manageRouter from './routes/manage.router';
|
||||
import miscRouter from './routes/misc.router';
|
||||
import oauthRouter from './routes/oauth-callback.router';
|
||||
import profileRouter from './routes/profile.router';
|
||||
import logsRouter from './routes/logs.router';
|
||||
import trackRouter from './routes/track.router';
|
||||
import webhookRouter from './routes/webhook.router';
|
||||
import { HttpError } from './utils/errors';
|
||||
@@ -209,6 +210,7 @@ const startServer = async () => {
|
||||
instance.register(importRouter, { prefix: '/import' });
|
||||
instance.register(insightsRouter, { prefix: '/insights' });
|
||||
instance.register(trackRouter, { prefix: '/track' });
|
||||
instance.register(logsRouter, { prefix: '/logs' });
|
||||
instance.register(manageRouter, { prefix: '/manage' });
|
||||
// Keep existing endpoints for backward compatibility
|
||||
instance.get('/healthcheck', healthcheck);
|
||||
|
||||
17
apps/api/src/routes/logs.router.ts
Normal file
17
apps/api/src/routes/logs.router.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import type { FastifyPluginCallback } from 'fastify';
|
||||
import { handler } from '@/controllers/logs.controller';
|
||||
import { clientHook } from '@/hooks/client.hook';
|
||||
import { duplicateHook } from '@/hooks/duplicate.hook';
|
||||
|
||||
const logsRouter: FastifyPluginCallback = async (fastify) => {
|
||||
fastify.addHook('preValidation', duplicateHook);
|
||||
fastify.addHook('preHandler', clientHook);
|
||||
|
||||
fastify.route({
|
||||
method: 'POST',
|
||||
url: '/',
|
||||
handler,
|
||||
});
|
||||
};
|
||||
|
||||
export default logsRouter;
|
||||
@@ -16,11 +16,11 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@nivo/funnel": "^0.99.0",
|
||||
"@number-flow/react": "0.5.10",
|
||||
"@opennextjs/cloudflare": "^1.17.1",
|
||||
"@number-flow/react": "0.6.0",
|
||||
"@opennextjs/cloudflare": "^1.18.0",
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/geo": "workspace:*",
|
||||
"@openpanel/nextjs": "^1.2.0",
|
||||
"@openpanel/nextjs": "^1.4.0",
|
||||
"@openpanel/payments": "workspace:^",
|
||||
"@openpanel/sdk-info": "workspace:^",
|
||||
"@openstatus/react": "0.0.3",
|
||||
@@ -28,37 +28,37 @@
|
||||
"@radix-ui/react-slider": "1.3.6",
|
||||
"@radix-ui/react-slot": "1.2.4",
|
||||
"@radix-ui/react-tooltip": "1.2.8",
|
||||
"cheerio": "^1.0.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"class-variance-authority": "0.7.1",
|
||||
"clsx": "2.1.1",
|
||||
"dotted-map": "2.2.3",
|
||||
"framer-motion": "12.23.25",
|
||||
"fumadocs-core": "16.2.2",
|
||||
"fumadocs-mdx": "14.0.4",
|
||||
"fumadocs-ui": "16.2.2",
|
||||
"geist": "1.5.1",
|
||||
"dotted-map": "3.1.0",
|
||||
"framer-motion": "12.38.0",
|
||||
"fumadocs-core": "16.7.7",
|
||||
"fumadocs-mdx": "14.2.11",
|
||||
"fumadocs-ui": "16.7.7",
|
||||
"geist": "1.7.0",
|
||||
"lucide-react": "^0.555.0",
|
||||
"next": "16.0.7",
|
||||
"next": "16.2.1",
|
||||
"next-themes": "^0.4.6",
|
||||
"react": "catalog:",
|
||||
"react-dom": "catalog:",
|
||||
"react-markdown": "^10.1.0",
|
||||
"recharts": "^2.15.0",
|
||||
"recharts": "^2.15.4",
|
||||
"rehype-external-links": "3.0.0",
|
||||
"tailwind-merge": "3.4.0",
|
||||
"tailwind-merge": "3.5.0",
|
||||
"tailwindcss-animate": "1.0.7",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4.1.17",
|
||||
"@tailwindcss/postcss": "^4.2.2",
|
||||
"@types/mdx": "^2.0.13",
|
||||
"@types/node": "catalog:",
|
||||
"@types/react": "catalog:",
|
||||
"@types/react-dom": "catalog:",
|
||||
"autoprefixer": "^10.4.22",
|
||||
"postcss": "^8.5.6",
|
||||
"tailwindcss": "4.1.17",
|
||||
"autoprefixer": "^10.4.27",
|
||||
"postcss": "^8.5.8",
|
||||
"tailwindcss": "4.2.2",
|
||||
"typescript": "catalog:",
|
||||
"wrangler": "^4.65.0"
|
||||
"wrangler": "^4.78.0"
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG NODE_VERSION=22.20.0
|
||||
ARG NODE_VERSION=22.22.2
|
||||
|
||||
FROM node:${NODE_VERSION}-slim AS base
|
||||
|
||||
|
||||
@@ -17,21 +17,21 @@
|
||||
"with-env": "dotenv -e ../../.env -c --"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/react": "^1.2.5",
|
||||
"@codemirror/commands": "^6.7.0",
|
||||
"@codemirror/lang-javascript": "^6.2.0",
|
||||
"@codemirror/lang-json": "^6.0.1",
|
||||
"@codemirror/state": "^6.4.0",
|
||||
"@ai-sdk/react": "^1.2.12",
|
||||
"@codemirror/commands": "^6.10.3",
|
||||
"@codemirror/lang-javascript": "^6.2.5",
|
||||
"@codemirror/lang-json": "^6.0.2",
|
||||
"@codemirror/state": "^6.6.0",
|
||||
"@codemirror/theme-one-dark": "^6.1.3",
|
||||
"@codemirror/view": "^6.35.0",
|
||||
"@codemirror/view": "^6.40.0",
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
"@dnd-kit/sortable": "^10.0.0",
|
||||
"@dnd-kit/utilities": "^3.2.2",
|
||||
"@faker-js/faker": "^9.6.0",
|
||||
"@hookform/resolvers": "^3.3.4",
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"@faker-js/faker": "^9.9.0",
|
||||
"@hookform/resolvers": "^3.10.0",
|
||||
"@hyperdx/node-opentelemetry": "^0.10.3",
|
||||
"@nivo/sankey": "^0.99.0",
|
||||
"@number-flow/react": "0.5.10",
|
||||
"@number-flow/react": "0.6.0",
|
||||
"@openpanel/common": "workspace:^",
|
||||
"@openpanel/constants": "workspace:^",
|
||||
"@openpanel/importer": "workspace:^",
|
||||
@@ -40,100 +40,100 @@
|
||||
"@openpanel/payments": "workspace:*",
|
||||
"@openpanel/sdk-info": "workspace:^",
|
||||
"@openpanel/validation": "workspace:^",
|
||||
"@openpanel/web": "^1.0.1",
|
||||
"@openpanel/web": "^1.3.0",
|
||||
"@radix-ui/react-accordion": "^1.2.12",
|
||||
"@radix-ui/react-alert-dialog": "^1.1.15",
|
||||
"@radix-ui/react-aspect-ratio": "^1.1.7",
|
||||
"@radix-ui/react-avatar": "^1.1.10",
|
||||
"@radix-ui/react-aspect-ratio": "^1.1.8",
|
||||
"@radix-ui/react-avatar": "^1.1.11",
|
||||
"@radix-ui/react-checkbox": "^1.3.3",
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
"@radix-ui/react-dropdown-menu": "^2.1.16",
|
||||
"@radix-ui/react-label": "^2.1.7",
|
||||
"@radix-ui/react-label": "^2.1.8",
|
||||
"@radix-ui/react-popover": "^1.1.15",
|
||||
"@radix-ui/react-portal": "^1.1.9",
|
||||
"@radix-ui/react-progress": "^1.1.7",
|
||||
"@radix-ui/react-portal": "^1.1.10",
|
||||
"@radix-ui/react-progress": "^1.1.8",
|
||||
"@radix-ui/react-radio-group": "^1.3.8",
|
||||
"@radix-ui/react-scroll-area": "^1.2.10",
|
||||
"@radix-ui/react-select": "^2.2.6",
|
||||
"@radix-ui/react-separator": "^1.1.7",
|
||||
"@radix-ui/react-slider": "1.2.3",
|
||||
"@radix-ui/react-slot": "^1.2.3",
|
||||
"@radix-ui/react-separator": "^1.1.8",
|
||||
"@radix-ui/react-slider": "1.3.6",
|
||||
"@radix-ui/react-slot": "^1.2.4",
|
||||
"@radix-ui/react-switch": "^1.2.6",
|
||||
"@radix-ui/react-tabs": "^1.1.13",
|
||||
"@radix-ui/react-toast": "^1.2.15",
|
||||
"@radix-ui/react-toggle": "^1.1.10",
|
||||
"@radix-ui/react-toggle-group": "^1.1.11",
|
||||
"@radix-ui/react-tooltip": "^1.2.8",
|
||||
"@reduxjs/toolkit": "^2.8.2",
|
||||
"@sentry/tanstackstart-react": "^9.12.0",
|
||||
"@reduxjs/toolkit": "^2.11.2",
|
||||
"@sentry/tanstackstart-react": "^10.46.0",
|
||||
"@tailwindcss/container-queries": "^0.1.1",
|
||||
"@tailwindcss/typography": "^0.5.15",
|
||||
"@tailwindcss/vite": "^4.0.6",
|
||||
"@tailwindcss/typography": "^0.5.19",
|
||||
"@tailwindcss/vite": "^4.2.2",
|
||||
"@tanstack/match-sorter-utils": "^8.19.4",
|
||||
"@tanstack/nitro-v2-vite-plugin": "^1.133.19",
|
||||
"@tanstack/react-devtools": "^0.7.6",
|
||||
"@tanstack/react-query": "^5.90.2",
|
||||
"@tanstack/react-query-devtools": "^5.90.2",
|
||||
"@tanstack/react-router": "^1.132.47",
|
||||
"@tanstack/react-router-devtools": "^1.132.51",
|
||||
"@tanstack/react-router-ssr-query": "^1.132.47",
|
||||
"@tanstack/react-start": "^1.132.56",
|
||||
"@tanstack/react-store": "^0.8.0",
|
||||
"@tanstack/nitro-v2-vite-plugin": "^1.154.9",
|
||||
"@tanstack/react-devtools": "^0.10.0",
|
||||
"@tanstack/react-query": "^5.95.2",
|
||||
"@tanstack/react-query-devtools": "^5.95.2",
|
||||
"@tanstack/react-router": "^1.168.10",
|
||||
"@tanstack/react-router-devtools": "^1.166.11",
|
||||
"@tanstack/react-router-ssr-query": "^1.166.10",
|
||||
"@tanstack/react-start": "^1.167.16",
|
||||
"@tanstack/react-store": "^0.9.3",
|
||||
"@tanstack/react-table": "^8.21.3",
|
||||
"@tanstack/react-virtual": "^3.13.12",
|
||||
"@tanstack/router-plugin": "^1.132.56",
|
||||
"@tanstack/store": "^0.8.0",
|
||||
"@trpc/client": "^11.6.0",
|
||||
"@trpc/react-query": "^11.6.0",
|
||||
"@trpc/server": "^11.6.0",
|
||||
"@trpc/tanstack-react-query": "^11.6.0",
|
||||
"@tanstack/react-virtual": "^3.13.23",
|
||||
"@tanstack/router-plugin": "^1.167.12",
|
||||
"@tanstack/store": "^0.9.3",
|
||||
"@trpc/client": "^11.16.0",
|
||||
"@trpc/react-query": "^11.16.0",
|
||||
"@trpc/server": "^11.16.0",
|
||||
"@trpc/tanstack-react-query": "^11.16.0",
|
||||
"@types/d3": "^7.4.3",
|
||||
"ai": "^4.2.10",
|
||||
"ai": "^4.3.19",
|
||||
"bind-event-listener": "^3.0.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^0.2.1",
|
||||
"codemirror": "^6.0.1",
|
||||
"d3": "^7.8.5",
|
||||
"date-fns": "^3.3.1",
|
||||
"debounce": "^2.2.0",
|
||||
"codemirror": "^6.0.2",
|
||||
"d3": "^7.9.0",
|
||||
"date-fns": "^3.6.0",
|
||||
"debounce": "^3.0.0",
|
||||
"embla-carousel-autoplay": "^8.6.0",
|
||||
"embla-carousel-react": "8.0.0-rc22",
|
||||
"flag-icons": "^7.1.0",
|
||||
"framer-motion": "^11.0.28",
|
||||
"hamburger-react": "^2.5.0",
|
||||
"input-otp": "^1.2.4",
|
||||
"javascript-time-ago": "^2.5.9",
|
||||
"katex": "^0.16.21",
|
||||
"embla-carousel-react": "8.6.0",
|
||||
"flag-icons": "^7.5.0",
|
||||
"framer-motion": "^12.38.0",
|
||||
"hamburger-react": "^2.5.2",
|
||||
"input-otp": "^1.4.2",
|
||||
"javascript-time-ago": "^2.6.4",
|
||||
"katex": "^0.16.44",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"lodash.isequal": "^4.5.0",
|
||||
"lodash.throttle": "^4.1.1",
|
||||
"lottie-react": "^2.4.0",
|
||||
"lottie-react": "^2.4.1",
|
||||
"lucide-react": "^0.476.0",
|
||||
"mitt": "^3.0.1",
|
||||
"nuqs": "^2.5.2",
|
||||
"nuqs": "^2.8.9",
|
||||
"prisma-error-enum": "^0.1.3",
|
||||
"pushmodal": "^1.0.3",
|
||||
"ramda": "^0.29.1",
|
||||
"pushmodal": "^1.0.5",
|
||||
"ramda": "^0.32.0",
|
||||
"random-animal-name": "^0.1.1",
|
||||
"rc-virtual-list": "^3.14.5",
|
||||
"rc-virtual-list": "^3.19.2",
|
||||
"react": "catalog:",
|
||||
"react-animate-height": "^3.2.3",
|
||||
"react-animated-numbers": "^1.1.1",
|
||||
"react-day-picker": "^9.9.0",
|
||||
"react-day-picker": "^9.14.0",
|
||||
"react-dom": "catalog:",
|
||||
"react-grid-layout": "^1.5.2",
|
||||
"react-hook-form": "^7.50.1",
|
||||
"react-in-viewport": "1.0.0-beta.8",
|
||||
"react-grid-layout": "^1.5.3",
|
||||
"react-hook-form": "^7.72.0",
|
||||
"react-in-viewport": "1.0.0-beta.9",
|
||||
"react-markdown": "^10.1.0",
|
||||
"react-redux": "^8.1.3",
|
||||
"react-resizable": "^3.0.5",
|
||||
"react-responsive": "^9.0.2",
|
||||
"react-resizable": "^3.1.3",
|
||||
"react-responsive": "^10.0.1",
|
||||
"react-simple-maps": "3.0.0",
|
||||
"react-svg-worldmap": "2.0.0-alpha.16",
|
||||
"react-syntax-highlighter": "^15.5.0",
|
||||
"react-use-websocket": "^4.7.0",
|
||||
"react-virtualized-auto-sizer": "^1.0.22",
|
||||
"react-svg-worldmap": "2.0.1",
|
||||
"react-syntax-highlighter": "^16.1.1",
|
||||
"react-use-websocket": "^4.13.0",
|
||||
"react-virtualized-auto-sizer": "^1.0.26",
|
||||
"recharts": "^2.15.4",
|
||||
"rehype-katex": "^7.0.1",
|
||||
"remark-gfm": "^4.0.1",
|
||||
@@ -142,42 +142,42 @@
|
||||
"remark-parse": "^11.0.0",
|
||||
"remark-rehype": "^11.1.2",
|
||||
"rrweb-player": "2.0.0-alpha.20",
|
||||
"short-unique-id": "^5.0.3",
|
||||
"slugify": "^1.6.6",
|
||||
"sonner": "^1.4.0",
|
||||
"short-unique-id": "^5.3.2",
|
||||
"slugify": "^1.6.8",
|
||||
"sonner": "^1.7.4",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^2.2.2",
|
||||
"tailwind-merge": "^3.0.2",
|
||||
"tailwindcss": "^4.0.6",
|
||||
"superjson": "^2.2.6",
|
||||
"tailwind-merge": "^3.5.0",
|
||||
"tailwindcss": "^4.2.2",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"tw-animate-css": "^1.3.6",
|
||||
"usehooks-ts": "^2.14.0",
|
||||
"vite-tsconfig-paths": "^5.1.4",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
"usehooks-ts": "^2.16.0",
|
||||
"vite-tsconfig-paths": "^6.1.1",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "1.9.4",
|
||||
"@cloudflare/vite-plugin": "1.20.3",
|
||||
"@biomejs/biome": "2.4.10",
|
||||
"@cloudflare/vite-plugin": "1.30.2",
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/trpc": "workspace:*",
|
||||
"@tanstack/devtools-event-client": "^0.3.3",
|
||||
"@testing-library/dom": "^10.4.0",
|
||||
"@testing-library/react": "^16.2.0",
|
||||
"@tanstack/devtools-event-client": "^0.4.3",
|
||||
"@testing-library/dom": "^10.4.1",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@types/lodash.debounce": "^4.0.9",
|
||||
"@types/lodash.isequal": "^4.5.8",
|
||||
"@types/lodash.throttle": "^4.1.9",
|
||||
"@types/ramda": "^0.31.0",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/react": "catalog:",
|
||||
"@types/react-dom": "catalog:",
|
||||
"@types/react-grid-layout": "^1.3.5",
|
||||
"@types/react-simple-maps": "^3.0.4",
|
||||
"@types/react-syntax-highlighter": "^15.5.11",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"jsdom": "^26.0.0",
|
||||
"@types/react-grid-layout": "^2.1.0",
|
||||
"@types/react-simple-maps": "^3.0.6",
|
||||
"@types/react-syntax-highlighter": "^15.5.13",
|
||||
"@vitejs/plugin-react": "^4.7.0",
|
||||
"jsdom": "^26.1.0",
|
||||
"typescript": "catalog:",
|
||||
"vite": "^6.3.5",
|
||||
"vitest": "^3.0.5",
|
||||
"web-vitals": "^4.2.4",
|
||||
"wrangler": "4.59.1"
|
||||
"vite": "^6.4.1",
|
||||
"vitest": "^3.2.4",
|
||||
"web-vitals": "^5.2.0",
|
||||
"wrangler": "4.78.0"
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
LayoutDashboardIcon,
|
||||
LayoutPanelTopIcon,
|
||||
PlusIcon,
|
||||
ScrollTextIcon,
|
||||
SearchIcon,
|
||||
SparklesIcon,
|
||||
TrendingUpDownIcon,
|
||||
@@ -61,6 +62,7 @@ export default function SidebarProjectMenu({
|
||||
<SidebarLink href={'/seo'} icon={SearchIcon} label="SEO" />
|
||||
<SidebarLink href={'/realtime'} icon={Globe2Icon} label="Realtime" />
|
||||
<SidebarLink href={'/events'} icon={GanttChartIcon} label="Events" />
|
||||
<SidebarLink href={'/logs'} icon={ScrollTextIcon} label="Logs" />
|
||||
<SidebarLink href={'/sessions'} icon={UsersIcon} label="Sessions" />
|
||||
<SidebarLink href={'/profiles'} icon={UserCircleIcon} label="Profiles" />
|
||||
<SidebarLink href={'/groups'} icon={Building2Icon} label="Groups" />
|
||||
|
||||
@@ -47,6 +47,7 @@ import { Route as AppOrganizationIdProjectIdReportsRouteImport } from './routes/
|
||||
import { Route as AppOrganizationIdProjectIdReferencesRouteImport } from './routes/_app.$organizationId.$projectId.references'
|
||||
import { Route as AppOrganizationIdProjectIdRealtimeRouteImport } from './routes/_app.$organizationId.$projectId.realtime'
|
||||
import { Route as AppOrganizationIdProjectIdPagesRouteImport } from './routes/_app.$organizationId.$projectId.pages'
|
||||
import { Route as AppOrganizationIdProjectIdLogsRouteImport } from './routes/_app.$organizationId.$projectId.logs'
|
||||
import { Route as AppOrganizationIdProjectIdInsightsRouteImport } from './routes/_app.$organizationId.$projectId.insights'
|
||||
import { Route as AppOrganizationIdProjectIdGroupsRouteImport } from './routes/_app.$organizationId.$projectId.groups'
|
||||
import { Route as AppOrganizationIdProjectIdDashboardsRouteImport } from './routes/_app.$organizationId.$projectId.dashboards'
|
||||
@@ -352,6 +353,12 @@ const AppOrganizationIdProjectIdPagesRoute =
|
||||
path: '/pages',
|
||||
getParentRoute: () => AppOrganizationIdProjectIdRoute,
|
||||
} as any)
|
||||
const AppOrganizationIdProjectIdLogsRoute =
|
||||
AppOrganizationIdProjectIdLogsRouteImport.update({
|
||||
id: '/logs',
|
||||
path: '/logs',
|
||||
getParentRoute: () => AppOrganizationIdProjectIdRoute,
|
||||
} as any)
|
||||
const AppOrganizationIdProjectIdInsightsRoute =
|
||||
AppOrganizationIdProjectIdInsightsRouteImport.update({
|
||||
id: '/insights',
|
||||
@@ -660,6 +667,7 @@ export interface FileRoutesByFullPath {
|
||||
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -738,6 +746,7 @@ export interface FileRoutesByTo {
|
||||
'/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -814,6 +823,7 @@ export interface FileRoutesById {
|
||||
'/_app/$organizationId/$projectId/dashboards': typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
'/_app/$organizationId/$projectId/groups': typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
'/_app/$organizationId/$projectId/insights': typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
'/_app/$organizationId/$projectId/logs': typeof AppOrganizationIdProjectIdLogsRoute
|
||||
'/_app/$organizationId/$projectId/pages': typeof AppOrganizationIdProjectIdPagesRoute
|
||||
'/_app/$organizationId/$projectId/realtime': typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
'/_app/$organizationId/$projectId/references': typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -905,6 +915,7 @@ export interface FileRouteTypes {
|
||||
| '/$organizationId/$projectId/dashboards'
|
||||
| '/$organizationId/$projectId/groups'
|
||||
| '/$organizationId/$projectId/insights'
|
||||
| '/$organizationId/$projectId/logs'
|
||||
| '/$organizationId/$projectId/pages'
|
||||
| '/$organizationId/$projectId/realtime'
|
||||
| '/$organizationId/$projectId/references'
|
||||
@@ -983,6 +994,7 @@ export interface FileRouteTypes {
|
||||
| '/$organizationId/$projectId/dashboards'
|
||||
| '/$organizationId/$projectId/groups'
|
||||
| '/$organizationId/$projectId/insights'
|
||||
| '/$organizationId/$projectId/logs'
|
||||
| '/$organizationId/$projectId/pages'
|
||||
| '/$organizationId/$projectId/realtime'
|
||||
| '/$organizationId/$projectId/references'
|
||||
@@ -1058,6 +1070,7 @@ export interface FileRouteTypes {
|
||||
| '/_app/$organizationId/$projectId/dashboards'
|
||||
| '/_app/$organizationId/$projectId/groups'
|
||||
| '/_app/$organizationId/$projectId/insights'
|
||||
| '/_app/$organizationId/$projectId/logs'
|
||||
| '/_app/$organizationId/$projectId/pages'
|
||||
| '/_app/$organizationId/$projectId/realtime'
|
||||
| '/_app/$organizationId/$projectId/references'
|
||||
@@ -1444,6 +1457,13 @@ declare module '@tanstack/react-router' {
|
||||
preLoaderRoute: typeof AppOrganizationIdProjectIdPagesRouteImport
|
||||
parentRoute: typeof AppOrganizationIdProjectIdRoute
|
||||
}
|
||||
'/_app/$organizationId/$projectId/logs': {
|
||||
id: '/_app/$organizationId/$projectId/logs'
|
||||
path: '/logs'
|
||||
fullPath: '/$organizationId/$projectId/logs'
|
||||
preLoaderRoute: typeof AppOrganizationIdProjectIdLogsRouteImport
|
||||
parentRoute: typeof AppOrganizationIdProjectIdRoute
|
||||
}
|
||||
'/_app/$organizationId/$projectId/insights': {
|
||||
id: '/_app/$organizationId/$projectId/insights'
|
||||
path: '/insights'
|
||||
@@ -2028,6 +2048,7 @@ interface AppOrganizationIdProjectIdRouteChildren {
|
||||
AppOrganizationIdProjectIdDashboardsRoute: typeof AppOrganizationIdProjectIdDashboardsRoute
|
||||
AppOrganizationIdProjectIdGroupsRoute: typeof AppOrganizationIdProjectIdGroupsRoute
|
||||
AppOrganizationIdProjectIdInsightsRoute: typeof AppOrganizationIdProjectIdInsightsRoute
|
||||
AppOrganizationIdProjectIdLogsRoute: typeof AppOrganizationIdProjectIdLogsRoute
|
||||
AppOrganizationIdProjectIdPagesRoute: typeof AppOrganizationIdProjectIdPagesRoute
|
||||
AppOrganizationIdProjectIdRealtimeRoute: typeof AppOrganizationIdProjectIdRealtimeRoute
|
||||
AppOrganizationIdProjectIdReferencesRoute: typeof AppOrganizationIdProjectIdReferencesRoute
|
||||
@@ -2054,6 +2075,7 @@ const AppOrganizationIdProjectIdRouteChildren: AppOrganizationIdProjectIdRouteCh
|
||||
AppOrganizationIdProjectIdGroupsRoute,
|
||||
AppOrganizationIdProjectIdInsightsRoute:
|
||||
AppOrganizationIdProjectIdInsightsRoute,
|
||||
AppOrganizationIdProjectIdLogsRoute: AppOrganizationIdProjectIdLogsRoute,
|
||||
AppOrganizationIdProjectIdPagesRoute: AppOrganizationIdProjectIdPagesRoute,
|
||||
AppOrganizationIdProjectIdRealtimeRoute:
|
||||
AppOrganizationIdProjectIdRealtimeRoute,
|
||||
|
||||
340
apps/start/src/routes/_app.$organizationId.$projectId.logs.tsx
Normal file
340
apps/start/src/routes/_app.$organizationId.$projectId.logs.tsx
Normal file
@@ -0,0 +1,340 @@
|
||||
import { useInfiniteQuery, useQuery } from '@tanstack/react-query';
|
||||
import { createFileRoute } from '@tanstack/react-router';
|
||||
import {
|
||||
AlertCircleIcon,
|
||||
AlertTriangleIcon,
|
||||
BugIcon,
|
||||
ChevronDownIcon,
|
||||
ChevronRightIcon,
|
||||
InfoIcon,
|
||||
SearchIcon,
|
||||
XCircleIcon,
|
||||
} from 'lucide-react';
|
||||
import { parseAsArrayOf, parseAsString, useQueryState } from 'nuqs';
|
||||
import { useState } from 'react';
|
||||
import { PageContainer } from '@/components/page-container';
|
||||
import { PageHeader } from '@/components/page-header';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { useTRPC } from '@/integrations/trpc/react';
|
||||
import { cn } from '@/utils/cn';
|
||||
import { createProjectTitle, PAGE_TITLES } from '@/utils/title';
|
||||
import type { IServiceLog } from '@openpanel/trpc';
|
||||
|
||||
export const Route = createFileRoute(
|
||||
'/_app/$organizationId/$projectId/logs',
|
||||
)({
|
||||
component: Component,
|
||||
head: () => ({
|
||||
meta: [{ title: createProjectTitle(PAGE_TITLES.LOGS) }],
|
||||
}),
|
||||
});
|
||||
|
||||
const SEVERITY_LEVELS = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'] as const;
|
||||
|
||||
type SeverityLevel = (typeof SEVERITY_LEVELS)[number];
|
||||
|
||||
function getSeverityVariant(
|
||||
severity: string,
|
||||
): 'default' | 'secondary' | 'info' | 'warning' | 'destructive' | 'outline' {
|
||||
switch (severity.toLowerCase()) {
|
||||
case 'fatal':
|
||||
case 'critical':
|
||||
case 'error':
|
||||
return 'destructive';
|
||||
case 'warn':
|
||||
case 'warning':
|
||||
return 'warning';
|
||||
case 'info':
|
||||
return 'info';
|
||||
default:
|
||||
return 'outline';
|
||||
}
|
||||
}
|
||||
|
||||
function SeverityIcon({ severity }: { severity: string }) {
|
||||
const s = severity.toLowerCase();
|
||||
const cls = 'size-3.5 shrink-0';
|
||||
if (s === 'fatal' || s === 'critical') return <XCircleIcon className={cn(cls, 'text-destructive')} />;
|
||||
if (s === 'error') return <AlertCircleIcon className={cn(cls, 'text-destructive')} />;
|
||||
if (s === 'warn' || s === 'warning') return <AlertTriangleIcon className={cn(cls, 'text-yellow-500')} />;
|
||||
if (s === 'debug' || s === 'trace') return <BugIcon className={cn(cls, 'text-muted-foreground')} />;
|
||||
return <InfoIcon className={cn(cls, 'text-blue-500')} />;
|
||||
}
|
||||
|
||||
function LogRow({ log }: { log: IServiceLog }) {
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const hasDetails =
|
||||
(log.attributes && Object.keys(log.attributes).length > 0) ||
|
||||
(log.resource && Object.keys(log.resource).length > 0) ||
|
||||
log.traceId ||
|
||||
log.loggerName;
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(
|
||||
'grid w-full grid-cols-[28px_90px_80px_1fr_120px] items-center gap-3 px-4 py-2.5 text-left text-sm transition-colors hover:bg-muted/50 border-b border-border/50',
|
||||
expanded && 'bg-muted/30',
|
||||
)}
|
||||
onClick={() => hasDetails && setExpanded((v) => !v)}
|
||||
>
|
||||
<span className="flex items-center justify-center text-muted-foreground">
|
||||
{hasDetails ? (
|
||||
expanded ? (
|
||||
<ChevronDownIcon className="size-3.5" />
|
||||
) : (
|
||||
<ChevronRightIcon className="size-3.5" />
|
||||
)
|
||||
) : null}
|
||||
</span>
|
||||
<span className="text-muted-foreground tabular-nums text-xs">
|
||||
{log.timestamp.toLocaleTimeString()}
|
||||
</span>
|
||||
<span>
|
||||
<Badge variant={getSeverityVariant(log.severityText)} className="gap-1 font-mono uppercase text-[10px]">
|
||||
<SeverityIcon severity={log.severityText} />
|
||||
{log.severityText}
|
||||
</Badge>
|
||||
</span>
|
||||
<span className="truncate font-mono text-xs">{log.body}</span>
|
||||
<span className="truncate text-xs text-muted-foreground text-right">
|
||||
{log.loggerName || log.os || log.device || '—'}
|
||||
</span>
|
||||
</button>
|
||||
|
||||
{expanded && (
|
||||
<div className="border-b border-border/50 bg-muted/20 px-4 py-3">
|
||||
<div className="grid gap-4 pl-[calc(28px+0.75rem)] text-xs sm:grid-cols-2">
|
||||
{log.loggerName && (
|
||||
<MetaRow label="Logger" value={log.loggerName} />
|
||||
)}
|
||||
{log.traceId && (
|
||||
<MetaRow label="Trace ID" value={log.traceId} mono />
|
||||
)}
|
||||
{log.spanId && (
|
||||
<MetaRow label="Span ID" value={log.spanId} mono />
|
||||
)}
|
||||
{log.deviceId && (
|
||||
<MetaRow label="Device ID" value={log.deviceId} mono />
|
||||
)}
|
||||
{log.os && (
|
||||
<MetaRow label="OS" value={`${log.os} ${log.osVersion}`.trim()} />
|
||||
)}
|
||||
{log.country && (
|
||||
<MetaRow label="Location" value={[log.city, log.country].filter(Boolean).join(', ')} />
|
||||
)}
|
||||
|
||||
{Object.keys(log.attributes).length > 0 && (
|
||||
<div className="col-span-2">
|
||||
<p className="mb-1.5 font-semibold text-muted-foreground">Attributes</p>
|
||||
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
|
||||
{Object.entries(log.attributes).map(([k, v]) => (
|
||||
<div key={k} className="flex gap-2">
|
||||
<span className="text-blue-500 shrink-0">{k}</span>
|
||||
<span className="text-muted-foreground">=</span>
|
||||
<span className="break-all">{v}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{Object.keys(log.resource).length > 0 && (
|
||||
<div className="col-span-2">
|
||||
<p className="mb-1.5 font-semibold text-muted-foreground">Resource</p>
|
||||
<div className="rounded-md border bg-background p-3 font-mono space-y-1">
|
||||
{Object.entries(log.resource).map(([k, v]) => (
|
||||
<div key={k} className="flex gap-2">
|
||||
<span className="text-emerald-600 shrink-0">{k}</span>
|
||||
<span className="text-muted-foreground">=</span>
|
||||
<span className="break-all">{v}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function MetaRow({ label, value, mono }: { label: string; value: string; mono?: boolean }) {
|
||||
return (
|
||||
<div className="flex gap-2">
|
||||
<span className="w-24 shrink-0 font-medium text-muted-foreground">{label}</span>
|
||||
<span className={cn('break-all', mono && 'font-mono')}>{value}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Component() {
|
||||
const { projectId } = Route.useParams();
|
||||
const trpc = useTRPC();
|
||||
|
||||
const [search, setSearch] = useQueryState('search', parseAsString.withDefault(''));
|
||||
const [severities, setSeverities] = useQueryState(
|
||||
'severity',
|
||||
parseAsArrayOf(parseAsString).withDefault([]),
|
||||
);
|
||||
|
||||
const countsQuery = useQuery(
|
||||
trpc.log.severityCounts.queryOptions({ projectId }),
|
||||
);
|
||||
|
||||
const logsQuery = useInfiniteQuery(
|
||||
trpc.log.list.infiniteQueryOptions(
|
||||
{
|
||||
projectId,
|
||||
search: search || undefined,
|
||||
severity: severities.length > 0 ? (severities as SeverityLevel[]) : undefined,
|
||||
take: 50,
|
||||
},
|
||||
{
|
||||
getNextPageParam: (lastPage) => lastPage.meta.next,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const logs = logsQuery.data?.pages.flatMap((p) => p.data) ?? [];
|
||||
const counts = countsQuery.data ?? {};
|
||||
|
||||
const toggleSeverity = (s: string) => {
|
||||
setSeverities((prev) =>
|
||||
prev.includes(s) ? prev.filter((x) => x !== s) : [...prev, s],
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<PageContainer>
|
||||
<PageHeader
|
||||
className="mb-6"
|
||||
title="Logs"
|
||||
description="Captured device and application logs in OpenTelemetry format"
|
||||
/>
|
||||
|
||||
{/* Severity filter chips */}
|
||||
<div className="mb-4 flex flex-wrap items-center gap-2">
|
||||
{SEVERITY_LEVELS.map((s) => {
|
||||
const active = severities.includes(s);
|
||||
const count = counts[s] ?? 0;
|
||||
return (
|
||||
<button
|
||||
key={s}
|
||||
type="button"
|
||||
onClick={() => toggleSeverity(s)}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1.5 rounded-full border px-3 py-1 text-xs font-medium transition-colors',
|
||||
active
|
||||
? 'border-foreground bg-foreground text-background'
|
||||
: 'border-border bg-transparent text-muted-foreground hover:border-foreground/50 hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
<SeverityIcon severity={s} />
|
||||
<span className="uppercase">{s}</span>
|
||||
{count > 0 && (
|
||||
<span className={cn('tabular-nums', active ? 'opacity-70' : 'opacity-50')}>
|
||||
{count.toLocaleString()}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
|
||||
{severities.length > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSeverities([])}
|
||||
className="text-xs text-muted-foreground hover:text-foreground underline"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Search */}
|
||||
<div className="mb-4 relative">
|
||||
<SearchIcon className="absolute left-3 top-1/2 -translate-y-1/2 size-4 text-muted-foreground" />
|
||||
<Input
|
||||
className="pl-9"
|
||||
placeholder="Search log messages…"
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value || null)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Log table */}
|
||||
<div className="rounded-lg border bg-card overflow-hidden">
|
||||
{/* Header */}
|
||||
<div className="grid grid-cols-[28px_90px_80px_1fr_120px] gap-3 border-b bg-muted/50 px-4 py-2 text-xs font-medium text-muted-foreground">
|
||||
<span />
|
||||
<span>Time</span>
|
||||
<span>Level</span>
|
||||
<span>Message</span>
|
||||
<span className="text-right">Source</span>
|
||||
</div>
|
||||
|
||||
{logsQuery.isPending && (
|
||||
<div className="space-y-0 divide-y">
|
||||
{Array.from({ length: 10 }).map((_, i) => (
|
||||
<div key={i} className="h-10 animate-pulse bg-muted/30" />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!logsQuery.isPending && logs.length === 0 && (
|
||||
<div className="flex flex-col items-center gap-2 py-16 text-center text-muted-foreground">
|
||||
<ScrollTextIcon className="size-10 opacity-30" />
|
||||
<p className="text-sm font-medium">No logs found</p>
|
||||
<p className="text-xs">
|
||||
{search || severities.length > 0
|
||||
? 'Try adjusting your filters'
|
||||
: 'Logs will appear here once your app starts sending them'}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{logs.map((log) => (
|
||||
<LogRow key={log.id} log={log} />
|
||||
))}
|
||||
|
||||
{logsQuery.hasNextPage && (
|
||||
<div className="flex justify-center p-4">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => logsQuery.fetchNextPage()}
|
||||
disabled={logsQuery.isFetchingNextPage}
|
||||
>
|
||||
{logsQuery.isFetchingNextPage ? 'Loading…' : 'Load more'}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</PageContainer>
|
||||
);
|
||||
}
|
||||
|
||||
function ScrollTextIcon({ className }: { className?: string }) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className={className}
|
||||
>
|
||||
<path d="M8 21h12a2 2 0 0 0 2-2v-2H10v2a2 2 0 1 1-4 0V5a2 2 0 1 0-4 0v3h4" />
|
||||
<path d="M19 17V5a2 2 0 0 0-2-2H4" />
|
||||
<path d="M15 8h-5" />
|
||||
<path d="M15 12h-5" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
@@ -88,6 +88,7 @@ export const PAGE_TITLES = {
|
||||
MEMBERS: 'Members',
|
||||
BILLING: 'Billing',
|
||||
CHAT: 'AI Assistant',
|
||||
LOGS: 'Logs',
|
||||
REALTIME: 'Realtime',
|
||||
REFERENCES: 'References',
|
||||
INSIGHTS: 'Insights',
|
||||
|
||||
@@ -10,15 +10,15 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/web": "workspace:*",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"react-router-dom": "^7.13.1"
|
||||
"react": "^19.2.4",
|
||||
"react-dom": "^19.2.4",
|
||||
"react-router-dom": "^7.13.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^4.7.0",
|
||||
"typescript": "catalog:",
|
||||
"vite": "^6.0.0"
|
||||
"vite": "^6.4.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG NODE_VERSION=22.20.0
|
||||
ARG NODE_VERSION=22.22.2
|
||||
|
||||
FROM node:${NODE_VERSION}-slim AS base
|
||||
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bull-board/api": "6.14.0",
|
||||
"@bull-board/express": "6.14.0",
|
||||
"@bull-board/api": "6.20.6",
|
||||
"@bull-board/express": "6.20.6",
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/email": "workspace:*",
|
||||
@@ -24,24 +24,25 @@
|
||||
"@openpanel/payments": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"bullmq": "^5.63.0",
|
||||
"date-fns": "^3.3.1",
|
||||
"express": "^4.18.2",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"bullmq": "^5.71.1",
|
||||
"date-fns": "^3.6.0",
|
||||
"express": "^4.22.1",
|
||||
"groupmq": "catalog:",
|
||||
"prom-client": "^15.1.3",
|
||||
"ramda": "^0.29.1",
|
||||
"ramda": "^0.32.0",
|
||||
"source-map-support": "^0.5.21",
|
||||
"sqlstring": "^2.3.3",
|
||||
"uuid": "^9.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/express": "^4.17.21",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/source-map-support": "^0.5.10",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"tsdown": "0.14.2",
|
||||
"@types/uuid": "^11.0.0",
|
||||
"tsdown": "0.21.7",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,11 @@ export async function bootCron() {
|
||||
type: 'flushGroups',
|
||||
pattern: 1000 * 10,
|
||||
},
|
||||
{
|
||||
name: 'flush',
|
||||
type: 'flushLogs',
|
||||
pattern: 1000 * 10,
|
||||
},
|
||||
{
|
||||
name: 'insightsDaily',
|
||||
type: 'insightsDaily',
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
gscQueue,
|
||||
importQueue,
|
||||
insightsQueue,
|
||||
logsQueue,
|
||||
miscQueue,
|
||||
notificationQueue,
|
||||
queueLogger,
|
||||
@@ -22,6 +23,7 @@ import { incomingEvent } from './jobs/events.incoming-event';
|
||||
import { gscJob } from './jobs/gsc';
|
||||
import { importJob } from './jobs/import';
|
||||
import { insightsProjectJob } from './jobs/insights';
|
||||
import { incomingLog } from './jobs/logs.incoming-log';
|
||||
import { miscJob } from './jobs/misc';
|
||||
import { notificationJob } from './jobs/notification';
|
||||
import { sessionsJob } from './jobs/sessions';
|
||||
@@ -59,6 +61,7 @@ function getEnabledQueues(): QueueName[] {
|
||||
'import',
|
||||
'insights',
|
||||
'gsc',
|
||||
'logs',
|
||||
];
|
||||
}
|
||||
|
||||
@@ -221,6 +224,20 @@ export function bootWorkers() {
|
||||
logger.info('Started worker for gsc', { concurrency });
|
||||
}
|
||||
|
||||
// Start logs worker
|
||||
if (enabledQueues.includes('logs')) {
|
||||
const concurrency = getConcurrencyFor('logs', 10);
|
||||
const logsWorker = new Worker(
|
||||
logsQueue.name,
|
||||
async (job) => {
|
||||
await incomingLog(job.data.payload);
|
||||
},
|
||||
{ ...workerOptions, concurrency },
|
||||
);
|
||||
workers.push(logsWorker);
|
||||
logger.info('Started worker for logs', { concurrency });
|
||||
}
|
||||
|
||||
if (workers.length === 0) {
|
||||
logger.warn(
|
||||
'No workers started. Check ENABLED_QUEUES environment variable.'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Job } from 'bullmq';
|
||||
|
||||
import { eventBuffer, groupBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
|
||||
import { eventBuffer, groupBuffer, logBuffer, profileBackfillBuffer, profileBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
|
||||
import type { CronQueuePayload } from '@openpanel/queue';
|
||||
|
||||
import { jobdeleteProjects } from './cron.delete-projects';
|
||||
@@ -33,6 +33,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {
|
||||
case 'flushGroups': {
|
||||
return await groupBuffer.tryFlush();
|
||||
}
|
||||
case 'flushLogs': {
|
||||
return await logBuffer.tryFlush();
|
||||
}
|
||||
case 'ping': {
|
||||
return await ping();
|
||||
}
|
||||
|
||||
63
apps/worker/src/jobs/logs.incoming-log.ts
Normal file
63
apps/worker/src/jobs/logs.incoming-log.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import type { IClickhouseLog } from '@openpanel/db';
|
||||
import { logBuffer } from '@openpanel/db';
|
||||
import type { LogsQueuePayload } from '@openpanel/queue';
|
||||
import { SEVERITY_TEXT_TO_NUMBER } from '@openpanel/validation';
|
||||
import { logger as baseLogger } from '@/utils/logger';
|
||||
|
||||
export async function incomingLog(
|
||||
payload: LogsQueuePayload['payload'],
|
||||
): Promise<void> {
|
||||
const logger = baseLogger.child({ projectId: payload.projectId });
|
||||
|
||||
try {
|
||||
const { log, uaInfo, geo, deviceId, sessionId, projectId, headers } = payload;
|
||||
|
||||
const sdkName = headers['openpanel-sdk-name'] ?? '';
|
||||
const sdkVersion = headers['openpanel-sdk-version'] ?? '';
|
||||
|
||||
const severityNumber =
|
||||
log.severityNumber ??
|
||||
SEVERITY_TEXT_TO_NUMBER[log.severity] ??
|
||||
9; // INFO fallback
|
||||
|
||||
const row: IClickhouseLog = {
|
||||
project_id: projectId,
|
||||
device_id: deviceId,
|
||||
profile_id: log.profileId ? String(log.profileId) : '',
|
||||
session_id: sessionId,
|
||||
timestamp: log.timestamp,
|
||||
observed_at: new Date().toISOString(),
|
||||
severity_number: severityNumber,
|
||||
severity_text: log.severity,
|
||||
body: log.body,
|
||||
trace_id: log.traceId ?? '',
|
||||
span_id: log.spanId ?? '',
|
||||
trace_flags: log.traceFlags ?? 0,
|
||||
logger_name: log.loggerName ?? '',
|
||||
attributes: log.attributes ?? {},
|
||||
resource: log.resource ?? {},
|
||||
sdk_name: sdkName,
|
||||
sdk_version: sdkVersion,
|
||||
country: geo.country ?? '',
|
||||
city: geo.city ?? '',
|
||||
region: geo.region ?? '',
|
||||
os: uaInfo.os ?? '',
|
||||
os_version: uaInfo.osVersion ?? '',
|
||||
browser: uaInfo.isServer ? '' : (uaInfo.browser ?? ''),
|
||||
browser_version: uaInfo.isServer ? '' : (uaInfo.browserVersion ?? ''),
|
||||
device: uaInfo.device ?? '',
|
||||
brand: uaInfo.isServer ? '' : (uaInfo.brand ?? ''),
|
||||
model: uaInfo.isServer ? '' : (uaInfo.model ?? ''),
|
||||
};
|
||||
|
||||
logBuffer.add(row);
|
||||
|
||||
logger.info('Log queued', {
|
||||
severity: log.severity,
|
||||
loggerName: log.loggerName,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to process incoming log', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
146
docker-compose.prod.yml
Normal file
146
docker-compose.prod.yml
Normal file
@@ -0,0 +1,146 @@
|
||||
services:
|
||||
op-db:
|
||||
image: postgres:18.3-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-db-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
environment:
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
op-kv:
|
||||
image: redis:8.6.2-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-kv-data:/data
|
||||
command: ["redis-server", "--maxmemory-policy", "noeviction"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
op-ch:
|
||||
image: clickhouse/clickhouse-server:26.3.2.3
|
||||
restart: always
|
||||
environment:
|
||||
- CLICKHOUSE_DEFAULT_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse}
|
||||
volumes:
|
||||
- op-ch-data:/var/lib/clickhouse
|
||||
- op-ch-logs:/var/log/clickhouse-server
|
||||
- ./self-hosting/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/op-config.xml:ro
|
||||
- ./self-hosting/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro
|
||||
- ./self-hosting/clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "clickhouse-client --query 'SELECT 1'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 262144
|
||||
hard: 262144
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
op-api:
|
||||
image: git.zias.be/zias/openpanel-api:latest
|
||||
restart: always
|
||||
ports:
|
||||
- "3001:3000"
|
||||
command: >
|
||||
sh -c "
|
||||
echo 'Running migrations...'
|
||||
CI=true pnpm -r run migrate:deploy
|
||||
|
||||
pnpm start
|
||||
"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
depends_on:
|
||||
op-db:
|
||||
condition: service_healthy
|
||||
op-ch:
|
||||
condition: service_healthy
|
||||
op-kv:
|
||||
condition: service_healthy
|
||||
env_file:
|
||||
- .env.prod
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "3"
|
||||
|
||||
op-dashboard:
|
||||
image: git.zias.be/zias/openpanel-dashboard:latest
|
||||
restart: always
|
||||
ports:
|
||||
- "3000:3000"
|
||||
depends_on:
|
||||
op-api:
|
||||
condition: service_healthy
|
||||
env_file:
|
||||
- .env.prod
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "3"
|
||||
|
||||
op-worker:
|
||||
image: git.zias.be/zias/openpanel-worker:latest
|
||||
restart: always
|
||||
ports:
|
||||
- "3002:3000"
|
||||
depends_on:
|
||||
op-api:
|
||||
condition: service_healthy
|
||||
env_file:
|
||||
- .env.prod
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "30m"
|
||||
max-file: "3"
|
||||
|
||||
volumes:
|
||||
op-db-data:
|
||||
driver: local
|
||||
op-kv-data:
|
||||
driver: local
|
||||
op-ch-data:
|
||||
driver: local
|
||||
op-ch-logs:
|
||||
driver: local
|
||||
@@ -2,7 +2,7 @@ version: "3"
|
||||
|
||||
services:
|
||||
op-db:
|
||||
image: postgres:14-alpine
|
||||
image: postgres:18.3-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- ./docker/data/op-db-data:/var/lib/postgresql/data
|
||||
@@ -13,7 +13,7 @@ services:
|
||||
- POSTGRES_PASSWORD=postgres
|
||||
|
||||
op-kv:
|
||||
image: redis:7.2.5-alpine
|
||||
image: redis:8.6.2-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- ./docker/data/op-kv-data:/data
|
||||
@@ -22,7 +22,7 @@ services:
|
||||
- 6379:6379
|
||||
|
||||
op-ch:
|
||||
image: clickhouse/clickhouse-server:26.1.3.52
|
||||
image: clickhouse/clickhouse-server:26.3.2.3
|
||||
restart: always
|
||||
volumes:
|
||||
- ./docker/data/op-ch-data:/var/lib/clickhouse
|
||||
|
||||
21
package.json
21
package.json
@@ -5,7 +5,7 @@
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"author": "Carl-Gerhard Lindesvärd",
|
||||
"packageManager": "pnpm@10.6.2",
|
||||
"packageManager": "pnpm@10.33.0",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"gen:bots": "pnpm -r --filter api gen:bots",
|
||||
@@ -30,11 +30,11 @@
|
||||
"pre-push": "[ -n \"$SKIP_HOOKS\" ] || (pnpm typecheck && pnpm test)"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"dotenv-cli": "^7.3.0",
|
||||
"semver": "^7.5.4",
|
||||
"@hyperdx/node-opentelemetry": "^0.10.3",
|
||||
"dotenv-cli": "^7.4.4",
|
||||
"semver": "^7.7.4",
|
||||
"typescript": "catalog:",
|
||||
"winston": "^3.14.2"
|
||||
"winston": "^3.19.0"
|
||||
},
|
||||
"trustedDependencies": [
|
||||
"@biomejs/biome",
|
||||
@@ -49,15 +49,16 @@
|
||||
"sharp"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.3.15",
|
||||
"@biomejs/biome": "2.4.10",
|
||||
"depcheck": "^1.4.7",
|
||||
"simple-git-hooks": "^2.12.1",
|
||||
"ultracite": "7.2.0",
|
||||
"vitest": "^3.0.4"
|
||||
"simple-git-hooks": "^2.13.1",
|
||||
"ultracite": "7.4.0",
|
||||
"vitest": "^3.2.4"
|
||||
},
|
||||
"pnpm": {
|
||||
"patchedDependencies": {
|
||||
"nuqs": "patches/nuqs.patch"
|
||||
"nuqs": "patches/nuqs.patch",
|
||||
"buffer-equal-constant-time": "patches/buffer-equal-constant-time.patch"
|
||||
},
|
||||
"overrides": {
|
||||
"rolldown": "1.0.0-beta.43",
|
||||
|
||||
@@ -12,13 +12,13 @@
|
||||
"@openpanel/validation": "workspace:^",
|
||||
"@oslojs/crypto": "^1.0.1",
|
||||
"@oslojs/encoding": "^1.1.0",
|
||||
"arctic": "^2.3.0"
|
||||
"arctic": "^2.3.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/react": "catalog:",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/constants": "workspace:*",
|
||||
"date-fns": "^3.3.1",
|
||||
"lru-cache": "^11.2.4",
|
||||
"date-fns": "^3.6.0",
|
||||
"lru-cache": "^11.2.7",
|
||||
"luxon": "^3.7.2",
|
||||
"mathjs": "^12.3.2",
|
||||
"nanoid": "^5.1.6",
|
||||
"ramda": "^0.29.1",
|
||||
"slugify": "^1.6.6",
|
||||
"mathjs": "^15.1.1",
|
||||
"nanoid": "^5.1.7",
|
||||
"ramda": "^0.32.0",
|
||||
"slugify": "^1.6.8",
|
||||
"superjson": "^1.13.3",
|
||||
"ua-parser-js": "^2.0.6",
|
||||
"ua-parser-js": "^2.0.9",
|
||||
"unique-names-generator": "^4.7.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -31,9 +31,9 @@
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@types/luxon": "^3.7.1",
|
||||
"@types/node": "catalog:",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/ua-parser-js": "^0.7.39",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"date-fns": "^3.3.1",
|
||||
"date-fns": "^3.6.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
72
packages/db/code-migrations/13-add-logs.ts
Normal file
72
packages/db/code-migrations/13-add-logs.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
import { createTable, runClickhouseMigrationCommands } from '../src/clickhouse/migration';
|
||||
import { getIsCluster, getIsSelfHosting, printBoxMessage } from './helpers';
|
||||
|
||||
export async function up() {
|
||||
const replicatedVersion = '1';
|
||||
const isClustered = getIsCluster();
|
||||
|
||||
const sqls: string[] = [];
|
||||
|
||||
sqls.push(
|
||||
...createTable({
|
||||
name: 'logs',
|
||||
columns: [
|
||||
'`id` UUID DEFAULT generateUUIDv4()',
|
||||
'`project_id` String CODEC(ZSTD(3))',
|
||||
'`device_id` String CODEC(ZSTD(3))',
|
||||
'`profile_id` String CODEC(ZSTD(3))',
|
||||
'`session_id` String CODEC(LZ4)',
|
||||
// OpenTelemetry log fields
|
||||
'`timestamp` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
|
||||
'`observed_at` DateTime64(9) CODEC(DoubleDelta, ZSTD(3))',
|
||||
'`severity_number` UInt8',
|
||||
'`severity_text` LowCardinality(String)',
|
||||
'`body` String CODEC(ZSTD(3))',
|
||||
'`trace_id` String CODEC(ZSTD(3))',
|
||||
'`span_id` String CODEC(ZSTD(3))',
|
||||
'`trace_flags` UInt32 DEFAULT 0',
|
||||
'`logger_name` LowCardinality(String)',
|
||||
// OTel attributes (log-level key-value pairs)
|
||||
'`attributes` Map(String, String) CODEC(ZSTD(3))',
|
||||
// OTel resource attributes (device/app metadata)
|
||||
'`resource` Map(String, String) CODEC(ZSTD(3))',
|
||||
// Server-enriched context
|
||||
'`sdk_name` LowCardinality(String)',
|
||||
'`sdk_version` LowCardinality(String)',
|
||||
'`country` LowCardinality(FixedString(2))',
|
||||
'`city` String',
|
||||
'`region` LowCardinality(String)',
|
||||
'`os` LowCardinality(String)',
|
||||
'`os_version` LowCardinality(String)',
|
||||
'`browser` LowCardinality(String)',
|
||||
'`browser_version` LowCardinality(String)',
|
||||
'`device` LowCardinality(String)',
|
||||
'`brand` LowCardinality(String)',
|
||||
'`model` LowCardinality(String)',
|
||||
],
|
||||
indices: [
|
||||
'INDEX idx_severity_number severity_number TYPE minmax GRANULARITY 1',
|
||||
'INDEX idx_body body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1',
|
||||
'INDEX idx_trace_id trace_id TYPE bloom_filter GRANULARITY 1',
|
||||
'INDEX idx_logger_name logger_name TYPE bloom_filter GRANULARITY 1',
|
||||
],
|
||||
orderBy: ['project_id', 'toDate(timestamp)', 'severity_number', 'device_id'],
|
||||
partitionBy: 'toYYYYMM(timestamp)',
|
||||
settings: {
|
||||
index_granularity: 8192,
|
||||
ttl_only_drop_parts: 1,
|
||||
},
|
||||
distributionHash: 'cityHash64(project_id, toString(toStartOfHour(timestamp)))',
|
||||
replicatedVersion,
|
||||
isClustered,
|
||||
}),
|
||||
);
|
||||
|
||||
printBoxMessage('Running migration: 13-add-logs', [
|
||||
'Creates the logs table for OpenTelemetry-compatible device/app log capture.',
|
||||
]);
|
||||
|
||||
if (!process.argv.includes('--dry')) {
|
||||
await runClickhouseMigrationCommands(sqls);
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@
|
||||
"with-env": "dotenv -e ../../.env -c --"
|
||||
},
|
||||
"dependencies": {
|
||||
"@clickhouse/client": "^1.12.1",
|
||||
"@clickhouse/client": "^1.18.2",
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/constants": "workspace:*",
|
||||
"@openpanel/json": "workspace:*",
|
||||
@@ -21,13 +21,13 @@
|
||||
"@openpanel/queue": "workspace:^",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@prisma/client": "^6.14.0",
|
||||
"@prisma/extension-read-replicas": "^0.4.1",
|
||||
"@prisma/client": "^6.19.2",
|
||||
"@prisma/extension-read-replicas": "^0.5.0",
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"jiti": "^2.4.1",
|
||||
"mathjs": "^12.3.2",
|
||||
"prisma-json-types-generator": "^3.1.1",
|
||||
"ramda": "^0.29.1",
|
||||
"jiti": "^2.6.1",
|
||||
"mathjs": "^15.1.1",
|
||||
"prisma-json-types-generator": "^3.6.2",
|
||||
"ramda": "^0.32.0",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"uuid": "^9.0.1",
|
||||
@@ -36,10 +36,10 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"prisma": "^6.14.0",
|
||||
"@types/uuid": "^11.0.0",
|
||||
"prisma": "^6.19.2",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { BotBuffer as BotBufferRedis } from './bot-buffer';
|
||||
import { EventBuffer as EventBufferRedis } from './event-buffer';
|
||||
import { GroupBuffer } from './group-buffer';
|
||||
import { LogBuffer } from './log-buffer';
|
||||
import { ProfileBackfillBuffer } from './profile-backfill-buffer';
|
||||
import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
|
||||
import { ReplayBuffer } from './replay-buffer';
|
||||
@@ -13,6 +14,8 @@ export const sessionBuffer = new SessionBuffer();
|
||||
export const profileBackfillBuffer = new ProfileBackfillBuffer();
|
||||
export const replayBuffer = new ReplayBuffer();
|
||||
export const groupBuffer = new GroupBuffer();
|
||||
export const logBuffer = new LogBuffer();
|
||||
|
||||
export type { ProfileBackfillEntry } from './profile-backfill-buffer';
|
||||
export type { IClickhouseSessionReplayChunk } from './replay-buffer';
|
||||
export type { IClickhouseLog } from './log-buffer';
|
||||
|
||||
193
packages/db/src/buffers/log-buffer.ts
Normal file
193
packages/db/src/buffers/log-buffer.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
import { getSafeJson } from '@openpanel/json';
|
||||
import { getRedisCache } from '@openpanel/redis';
|
||||
import { ch } from '../clickhouse/client';
|
||||
import { BaseBuffer } from './base-buffer';
|
||||
|
||||
export interface IClickhouseLog {
|
||||
id?: string;
|
||||
project_id: string;
|
||||
device_id: string;
|
||||
profile_id: string;
|
||||
session_id: string;
|
||||
timestamp: string;
|
||||
observed_at: string;
|
||||
severity_number: number;
|
||||
severity_text: string;
|
||||
body: string;
|
||||
trace_id: string;
|
||||
span_id: string;
|
||||
trace_flags: number;
|
||||
logger_name: string;
|
||||
attributes: Record<string, string>;
|
||||
resource: Record<string, string>;
|
||||
sdk_name: string;
|
||||
sdk_version: string;
|
||||
country: string;
|
||||
city: string;
|
||||
region: string;
|
||||
os: string;
|
||||
os_version: string;
|
||||
browser: string;
|
||||
browser_version: string;
|
||||
device: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export class LogBuffer extends BaseBuffer {
|
||||
private batchSize = process.env.LOG_BUFFER_BATCH_SIZE
|
||||
? Number.parseInt(process.env.LOG_BUFFER_BATCH_SIZE, 10)
|
||||
: 4000;
|
||||
private chunkSize = process.env.LOG_BUFFER_CHUNK_SIZE
|
||||
? Number.parseInt(process.env.LOG_BUFFER_CHUNK_SIZE, 10)
|
||||
: 1000;
|
||||
private microBatchIntervalMs = process.env.LOG_BUFFER_MICRO_BATCH_MS
|
||||
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_MS, 10)
|
||||
: 10;
|
||||
private microBatchMaxSize = process.env.LOG_BUFFER_MICRO_BATCH_SIZE
|
||||
? Number.parseInt(process.env.LOG_BUFFER_MICRO_BATCH_SIZE, 10)
|
||||
: 100;
|
||||
|
||||
private pendingLogs: IClickhouseLog[] = [];
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private isFlushing = false;
|
||||
private flushRetryCount = 0;
|
||||
|
||||
private queueKey = 'log_buffer:queue';
|
||||
protected bufferCounterKey = 'log_buffer:total_count';
|
||||
|
||||
constructor() {
|
||||
super({
|
||||
name: 'log',
|
||||
onFlush: async () => {
|
||||
await this.processBuffer();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
add(log: IClickhouseLog) {
|
||||
this.pendingLogs.push(log);
|
||||
|
||||
if (this.pendingLogs.length >= this.microBatchMaxSize) {
|
||||
this.flushLocalBuffer();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.flushTimer) {
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flushTimer = null;
|
||||
this.flushLocalBuffer();
|
||||
}, this.microBatchIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
public async flush() {
|
||||
if (this.flushTimer) {
|
||||
clearTimeout(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
await this.flushLocalBuffer();
|
||||
}
|
||||
|
||||
private async flushLocalBuffer() {
|
||||
if (this.isFlushing || this.pendingLogs.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isFlushing = true;
|
||||
const logsToFlush = this.pendingLogs;
|
||||
this.pendingLogs = [];
|
||||
|
||||
try {
|
||||
const redis = getRedisCache();
|
||||
const multi = redis.multi();
|
||||
for (const log of logsToFlush) {
|
||||
multi.rpush(this.queueKey, JSON.stringify(log));
|
||||
}
|
||||
multi.incrby(this.bufferCounterKey, logsToFlush.length);
|
||||
await multi.exec();
|
||||
this.flushRetryCount = 0;
|
||||
} catch (error) {
|
||||
this.pendingLogs = logsToFlush.concat(this.pendingLogs);
|
||||
this.flushRetryCount += 1;
|
||||
this.logger.warn('Failed to flush log buffer to Redis; logs re-queued', {
|
||||
error,
|
||||
logCount: logsToFlush.length,
|
||||
flushRetryCount: this.flushRetryCount,
|
||||
});
|
||||
} finally {
|
||||
this.isFlushing = false;
|
||||
if (this.pendingLogs.length > 0 && !this.flushTimer) {
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flushTimer = null;
|
||||
this.flushLocalBuffer();
|
||||
}, this.microBatchIntervalMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async processBuffer() {
|
||||
const redis = getRedisCache();
|
||||
|
||||
try {
|
||||
const queueLogs = await redis.lrange(this.queueKey, 0, this.batchSize - 1);
|
||||
|
||||
if (queueLogs.length === 0) {
|
||||
this.logger.debug('No logs to process');
|
||||
return;
|
||||
}
|
||||
|
||||
const logsToClickhouse: IClickhouseLog[] = [];
|
||||
for (const logStr of queueLogs) {
|
||||
const log = getSafeJson<IClickhouseLog>(logStr);
|
||||
if (log) {
|
||||
logsToClickhouse.push(log);
|
||||
}
|
||||
}
|
||||
|
||||
if (logsToClickhouse.length === 0) {
|
||||
this.logger.debug('No valid logs to process');
|
||||
return;
|
||||
}
|
||||
|
||||
logsToClickhouse.sort(
|
||||
(a, b) =>
|
||||
new Date(a.timestamp || 0).getTime() -
|
||||
new Date(b.timestamp || 0).getTime(),
|
||||
);
|
||||
|
||||
this.logger.info('Inserting logs into ClickHouse', {
|
||||
totalLogs: logsToClickhouse.length,
|
||||
chunks: Math.ceil(logsToClickhouse.length / this.chunkSize),
|
||||
});
|
||||
|
||||
for (const chunk of this.chunks(logsToClickhouse, this.chunkSize)) {
|
||||
await ch.insert({
|
||||
table: 'logs',
|
||||
values: chunk,
|
||||
format: 'JSONEachRow',
|
||||
});
|
||||
}
|
||||
|
||||
await redis
|
||||
.multi()
|
||||
.ltrim(this.queueKey, queueLogs.length, -1)
|
||||
.decrby(this.bufferCounterKey, queueLogs.length)
|
||||
.exec();
|
||||
|
||||
this.logger.info('Processed logs from Redis buffer', {
|
||||
batchSize: this.batchSize,
|
||||
logsProcessed: logsToClickhouse.length,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('Error processing log Redis buffer', { error });
|
||||
}
|
||||
}
|
||||
|
||||
public getBufferSize() {
|
||||
return this.getBufferSizeWithCounter(async () => {
|
||||
const redis = getRedisCache();
|
||||
return await redis.llen(this.queueKey);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -9,10 +9,10 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@react-email/components": "^0.5.6",
|
||||
"@react-email/components": "^1.0.10",
|
||||
"react": "catalog:",
|
||||
"react-dom": "catalog:",
|
||||
"resend": "^4.0.1",
|
||||
"resend": "^4.8.0",
|
||||
"responsive-react-email": "^0.0.5",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
|
||||
@@ -7,15 +7,15 @@
|
||||
"codegen": "jiti scripts/download.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@maxmind/geoip2-node": "^6.1.0",
|
||||
"lru-cache": "^11.2.2"
|
||||
"@maxmind/geoip2-node": "^6.3.4",
|
||||
"lru-cache": "^11.2.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"fast-extract": "^1.4.3",
|
||||
"jiti": "^2.4.1",
|
||||
"tar": "^7.4.3",
|
||||
"fast-extract": "^1.14.2",
|
||||
"jiti": "^2.6.1",
|
||||
"tar": "^7.5.13",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,18 +18,18 @@
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"csv-parse": "^6.1.0",
|
||||
"ramda": "^0.29.1",
|
||||
"csv-parse": "^6.2.1",
|
||||
"ramda": "^0.32.0",
|
||||
"uuid": "^9.0.1",
|
||||
"zod": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/logger": "workspace:*",
|
||||
"@types/node": "^20.0.0",
|
||||
"@types/node": "^20.19.37",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/uuid": "^9.0.7",
|
||||
"bullmq": "^5.8.7",
|
||||
"typescript": "^5.0.0",
|
||||
"vitest": "^1.0.0"
|
||||
"@types/uuid": "^11.0.0",
|
||||
"bullmq": "^5.71.1",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^1.6.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@slack/bolt": "^3.18.0",
|
||||
"@slack/oauth": "^3.0.0"
|
||||
"@slack/bolt": "^3.22.0",
|
||||
"@slack/oauth": "^3.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
|
||||
@@ -11,11 +11,11 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@babel/parser": "^7.26.0"
|
||||
"@babel/parser": "^7.29.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"typescript": "catalog:",
|
||||
"vitest": "^2.1.8"
|
||||
"vitest": "^2.1.9"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,13 +7,13 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hyperdx/node-opentelemetry": "^0.8.1",
|
||||
"winston": "^3.14.2"
|
||||
"@hyperdx/node-opentelemetry": "^0.10.3",
|
||||
"winston": "^3.19.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"date-fns": "^3.3.1",
|
||||
"prisma": "^5.1.1",
|
||||
"date-fns": "^3.6.0",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,16 +11,16 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@polar-sh/sdk": "^0.35.4"
|
||||
"@polar-sh/sdk": "^0.46.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/inquirer": "^9.0.7",
|
||||
"@types/inquirer": "^9.0.9",
|
||||
"@types/inquirer-autocomplete-prompt": "^3.0.3",
|
||||
"@types/node": "catalog:",
|
||||
"@types/react": "catalog:",
|
||||
"inquirer": "^9.3.5",
|
||||
"inquirer": "^9.3.8",
|
||||
"inquirer-autocomplete-prompt": "^3.0.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/logger": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"bullmq": "^5.63.0",
|
||||
"bullmq": "^5.71.1",
|
||||
"groupmq": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -8,7 +8,7 @@ import { createLogger } from '@openpanel/logger';
|
||||
import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
|
||||
import { Queue } from 'bullmq';
|
||||
import { Queue as GroupQueue } from 'groupmq';
|
||||
import type { ITrackPayload } from '../../validation';
|
||||
import type { ILogPayload, ITrackPayload } from '../../validation';
|
||||
|
||||
export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(
|
||||
process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',
|
||||
@@ -138,6 +138,10 @@ export type CronQueuePayloadFlushGroups = {
|
||||
type: 'flushGroups';
|
||||
payload: undefined;
|
||||
};
|
||||
export type CronQueuePayloadFlushLogs = {
|
||||
type: 'flushLogs';
|
||||
payload: undefined;
|
||||
};
|
||||
export type CronQueuePayload =
|
||||
| CronQueuePayloadSalt
|
||||
| CronQueuePayloadFlushEvents
|
||||
@@ -146,6 +150,7 @@ export type CronQueuePayload =
|
||||
| CronQueuePayloadFlushProfileBackfill
|
||||
| CronQueuePayloadFlushReplay
|
||||
| CronQueuePayloadFlushGroups
|
||||
| CronQueuePayloadFlushLogs
|
||||
| CronQueuePayloadPing
|
||||
| CronQueuePayloadProject
|
||||
| CronQueuePayloadInsightsDaily
|
||||
@@ -297,3 +302,50 @@ export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {
|
||||
removeOnFail: 100,
|
||||
},
|
||||
});
|
||||
|
||||
export type LogsQueuePayload = {
|
||||
type: 'incomingLog';
|
||||
payload: {
|
||||
projectId: string;
|
||||
log: ILogPayload & {
|
||||
timestamp: string;
|
||||
};
|
||||
uaInfo:
|
||||
| {
|
||||
readonly isServer: true;
|
||||
readonly device: 'server';
|
||||
readonly os: '';
|
||||
readonly osVersion: '';
|
||||
readonly browser: '';
|
||||
readonly browserVersion: '';
|
||||
readonly brand: '';
|
||||
readonly model: '';
|
||||
}
|
||||
| {
|
||||
readonly os: string | undefined;
|
||||
readonly osVersion: string | undefined;
|
||||
readonly browser: string | undefined;
|
||||
readonly browserVersion: string | undefined;
|
||||
readonly device: string;
|
||||
readonly brand: string | undefined;
|
||||
readonly model: string | undefined;
|
||||
readonly isServer: false;
|
||||
};
|
||||
geo: {
|
||||
country: string | undefined;
|
||||
city: string | undefined;
|
||||
region: string | undefined;
|
||||
};
|
||||
headers: Record<string, string | undefined>;
|
||||
deviceId: string;
|
||||
sessionId: string;
|
||||
};
|
||||
};
|
||||
|
||||
export const logsQueue = new Queue<LogsQueuePayload>(getQueueName('logs'), {
|
||||
connection: getRedisQueue(),
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 1000,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -8,14 +8,14 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/json": "workspace:*",
|
||||
"ioredis": "5.8.2",
|
||||
"lru-cache": "^11.2.2"
|
||||
"ioredis": "5.10.1",
|
||||
"lru-cache": "^11.2.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/db": "workspace:*",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
"react": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "catalog:",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"prisma": "^5.1.1",
|
||||
"@types/react": "catalog:",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"@openpanel/web": "workspace:1.3.0-local"
|
||||
},
|
||||
"devDependencies": {
|
||||
"astro": "^5.7.7"
|
||||
"astro": "^5.18.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"astro": "^4.0.0 || ^5.0.0"
|
||||
|
||||
@@ -10,17 +10,17 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@openpanel/sdk": "workspace:1.3.0-local",
|
||||
"@openpanel/common": "workspace:*"
|
||||
"@openpanel/common": "workspace:*",
|
||||
"@openpanel/sdk": "workspace:1.3.0-local"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"express": "^4.17.0 || ^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/express": "^5.0.3",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/request-ip": "^0.0.41",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/react": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,12 +31,12 @@
|
||||
"nuxt": "^3.0.0 || ^4.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nuxt/kit": "^3.0.0",
|
||||
"@nuxt/kit": "^3.21.2",
|
||||
"@nuxt/module-builder": "^1.0.2",
|
||||
"@nuxt/types": "^2.18.1",
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@vue/runtime-core": "^3.5.25",
|
||||
"@vue/runtime-core": "^3.5.31",
|
||||
"typescript": "catalog:",
|
||||
"unbuild": "^3.6.1"
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -9,12 +9,11 @@
|
||||
"build": "rm -rf dist && tsup",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
IGroupPayload as GroupPayload,
|
||||
IIdentifyPayload as IdentifyPayload,
|
||||
IIncrementPayload as IncrementPayload,
|
||||
ISeverityText,
|
||||
ITrackHandlerPayload as TrackHandlerPayload,
|
||||
ITrackPayload as TrackPayload,
|
||||
} from '@openpanel/validation';
|
||||
@@ -19,6 +20,7 @@ export type {
|
||||
GroupPayload,
|
||||
IdentifyPayload,
|
||||
IncrementPayload,
|
||||
ISeverityText,
|
||||
TrackHandlerPayload,
|
||||
TrackPayload,
|
||||
};
|
||||
@@ -29,6 +31,33 @@ export interface TrackProperties {
|
||||
groups?: string[];
|
||||
}
|
||||
|
||||
export interface LogProperties {
|
||||
/** Logger name (e.g. "com.example.MyActivity") */
|
||||
loggerName?: string;
|
||||
traceId?: string;
|
||||
spanId?: string;
|
||||
traceFlags?: number;
|
||||
/** Log-level key-value attributes */
|
||||
attributes?: Record<string, string>;
|
||||
/** Resource/device attributes */
|
||||
resource?: Record<string, string>;
|
||||
/** ISO 8601 timestamp; defaults to now */
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
interface LogPayloadForQueue {
|
||||
body: string;
|
||||
severity: ISeverityText;
|
||||
loggerName?: string;
|
||||
traceId?: string;
|
||||
spanId?: string;
|
||||
traceFlags?: number;
|
||||
attributes?: Record<string, string>;
|
||||
resource?: Record<string, string>;
|
||||
timestamp: string;
|
||||
profileId?: string;
|
||||
}
|
||||
|
||||
export type UpsertGroupPayload = GroupPayload;
|
||||
|
||||
export interface OpenPanelOptions {
|
||||
@@ -57,6 +86,10 @@ export class OpenPanel {
|
||||
sessionId?: string;
|
||||
global?: Record<string, unknown>;
|
||||
queue: TrackHandlerPayload[] = [];
|
||||
private logQueue: LogPayloadForQueue[] = [];
|
||||
private logFlushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private logFlushIntervalMs = 2000;
|
||||
private logFlushMaxSize = 50;
|
||||
|
||||
constructor(options: OpenPanelOptions) {
|
||||
this.options = options;
|
||||
@@ -327,6 +360,67 @@ export class OpenPanel {
|
||||
this.queue = remaining;
|
||||
}
|
||||
|
||||
captureLog(
|
||||
severity: ISeverityText,
|
||||
body: string,
|
||||
properties?: LogProperties,
|
||||
) {
|
||||
if (this.options.disabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: LogPayloadForQueue = {
|
||||
body,
|
||||
severity,
|
||||
timestamp: properties?.timestamp ?? new Date().toISOString(),
|
||||
...(this.profileId ? { profileId: this.profileId } : {}),
|
||||
...(properties?.loggerName ? { loggerName: properties.loggerName } : {}),
|
||||
...(properties?.traceId ? { traceId: properties.traceId } : {}),
|
||||
...(properties?.spanId ? { spanId: properties.spanId } : {}),
|
||||
...(properties?.traceFlags !== undefined
|
||||
? { traceFlags: properties.traceFlags }
|
||||
: {}),
|
||||
...(properties?.attributes ? { attributes: properties.attributes } : {}),
|
||||
...(properties?.resource ? { resource: properties.resource } : {}),
|
||||
};
|
||||
|
||||
this.logQueue.push(entry);
|
||||
|
||||
if (this.logQueue.length >= this.logFlushMaxSize) {
|
||||
this.flushLogs();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.logFlushTimer) {
|
||||
this.logFlushTimer = setTimeout(() => {
|
||||
this.logFlushTimer = null;
|
||||
this.flushLogs();
|
||||
}, this.logFlushIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private async flushLogs() {
|
||||
if (this.logFlushTimer) {
|
||||
clearTimeout(this.logFlushTimer);
|
||||
this.logFlushTimer = null;
|
||||
}
|
||||
|
||||
if (this.logQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const batch = this.logQueue;
|
||||
this.logQueue = [];
|
||||
|
||||
try {
|
||||
await this.api.fetch('/logs', { logs: batch });
|
||||
} catch (error) {
|
||||
this.log('Failed to flush logs', error);
|
||||
// Re-queue on failure
|
||||
this.logQueue = batch.concat(this.logQueue);
|
||||
}
|
||||
}
|
||||
|
||||
log(...args: any[]) {
|
||||
if (this.options.debug) {
|
||||
console.log('[OpenPanel.dev]', ...args);
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"tsup": "^7.2.0",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export * from './src/root';
|
||||
export * from './src/trpc';
|
||||
export { getProjectAccess } from './src/access';
|
||||
export type { IServiceLog } from './src/routers/log';
|
||||
|
||||
@@ -16,17 +16,17 @@
|
||||
"@openpanel/integrations": "workspace:^",
|
||||
"@openpanel/js-runtime": "workspace:*",
|
||||
"@openpanel/payments": "workspace:^",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@openpanel/redis": "workspace:*",
|
||||
"@openpanel/validation": "workspace:*",
|
||||
"@openpanel/queue": "workspace:*",
|
||||
"@trpc-limiter/redis": "^0.0.2",
|
||||
"@trpc/client": "^11.6.0",
|
||||
"@trpc/server": "^11.6.0",
|
||||
"date-fns": "^3.3.1",
|
||||
"mathjs": "^12.3.2",
|
||||
"@trpc/client": "^11.16.0",
|
||||
"@trpc/server": "^11.16.0",
|
||||
"date-fns": "^3.6.0",
|
||||
"mathjs": "^15.1.1",
|
||||
"prisma-error-enum": "^0.1.3",
|
||||
"ramda": "^0.29.1",
|
||||
"short-unique-id": "^5.0.3",
|
||||
"ramda": "^0.32.0",
|
||||
"short-unique-id": "^5.3.2",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"uuid": "^9.0.1",
|
||||
@@ -35,9 +35,9 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/ramda": "^0.29.6",
|
||||
"@types/ramda": "^0.31.1",
|
||||
"@types/sqlstring": "^2.3.2",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import { eventRouter } from './routers/event';
|
||||
import { groupRouter } from './routers/group';
|
||||
import { gscRouter } from './routers/gsc';
|
||||
import { importRouter } from './routers/import';
|
||||
import { logRouter } from './routers/log';
|
||||
import { insightRouter } from './routers/insight';
|
||||
import { integrationRouter } from './routers/integration';
|
||||
import { notificationRouter } from './routers/notification';
|
||||
@@ -57,6 +58,7 @@ export const appRouter = createTRPCRouter({
|
||||
email: emailRouter,
|
||||
gsc: gscRouter,
|
||||
group: groupRouter,
|
||||
log: logRouter,
|
||||
});
|
||||
|
||||
// export type definition of API
|
||||
|
||||
212
packages/trpc/src/routers/log.ts
Normal file
212
packages/trpc/src/routers/log.ts
Normal file
@@ -0,0 +1,212 @@
|
||||
import { chQuery, convertClickhouseDateToJs } from '@openpanel/db';
|
||||
import { zSeverityText } from '@openpanel/validation';
|
||||
import sqlstring from 'sqlstring';
|
||||
import { z } from 'zod';
|
||||
import { createTRPCRouter, protectedProcedure } from '../trpc';
|
||||
|
||||
export interface IServiceLog {
|
||||
id: string;
|
||||
projectId: string;
|
||||
deviceId: string;
|
||||
profileId: string;
|
||||
sessionId: string;
|
||||
timestamp: Date;
|
||||
severityNumber: number;
|
||||
severityText: string;
|
||||
body: string;
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
traceFlags: number;
|
||||
loggerName: string;
|
||||
attributes: Record<string, string>;
|
||||
resource: Record<string, string>;
|
||||
sdkName: string;
|
||||
sdkVersion: string;
|
||||
country: string;
|
||||
city: string;
|
||||
region: string;
|
||||
os: string;
|
||||
osVersion: string;
|
||||
browser: string;
|
||||
browserVersion: string;
|
||||
device: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
interface IClickhouseLog {
|
||||
id: string;
|
||||
project_id: string;
|
||||
device_id: string;
|
||||
profile_id: string;
|
||||
session_id: string;
|
||||
timestamp: string;
|
||||
severity_number: number;
|
||||
severity_text: string;
|
||||
body: string;
|
||||
trace_id: string;
|
||||
span_id: string;
|
||||
trace_flags: number;
|
||||
logger_name: string;
|
||||
attributes: Record<string, string>;
|
||||
resource: Record<string, string>;
|
||||
sdk_name: string;
|
||||
sdk_version: string;
|
||||
country: string;
|
||||
city: string;
|
||||
region: string;
|
||||
os: string;
|
||||
os_version: string;
|
||||
browser: string;
|
||||
browser_version: string;
|
||||
device: string;
|
||||
brand: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
function toServiceLog(row: IClickhouseLog): IServiceLog {
|
||||
return {
|
||||
id: row.id,
|
||||
projectId: row.project_id,
|
||||
deviceId: row.device_id,
|
||||
profileId: row.profile_id,
|
||||
sessionId: row.session_id,
|
||||
timestamp: convertClickhouseDateToJs(row.timestamp),
|
||||
severityNumber: row.severity_number,
|
||||
severityText: row.severity_text,
|
||||
body: row.body,
|
||||
traceId: row.trace_id,
|
||||
spanId: row.span_id,
|
||||
traceFlags: row.trace_flags,
|
||||
loggerName: row.logger_name,
|
||||
attributes: row.attributes,
|
||||
resource: row.resource,
|
||||
sdkName: row.sdk_name,
|
||||
sdkVersion: row.sdk_version,
|
||||
country: row.country,
|
||||
city: row.city,
|
||||
region: row.region,
|
||||
os: row.os,
|
||||
osVersion: row.os_version,
|
||||
browser: row.browser,
|
||||
browserVersion: row.browser_version,
|
||||
device: row.device,
|
||||
brand: row.brand,
|
||||
model: row.model,
|
||||
};
|
||||
}
|
||||
|
||||
export const logRouter = createTRPCRouter({
|
||||
list: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
projectId: z.string(),
|
||||
cursor: z.string().nullish(),
|
||||
severity: z.array(zSeverityText).optional(),
|
||||
search: z.string().optional(),
|
||||
loggerName: z.string().optional(),
|
||||
startDate: z.date().optional(),
|
||||
endDate: z.date().optional(),
|
||||
take: z.number().default(50),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input }) => {
|
||||
const { projectId, cursor, severity, search, loggerName, startDate, endDate, take } = input;
|
||||
|
||||
const conditions: string[] = [
|
||||
`project_id = ${sqlstring.escape(projectId)}`,
|
||||
];
|
||||
|
||||
if (cursor) {
|
||||
conditions.push(`timestamp < ${sqlstring.escape(cursor)}`);
|
||||
}
|
||||
|
||||
if (severity && severity.length > 0) {
|
||||
const escaped = severity.map((s) => sqlstring.escape(s)).join(', ');
|
||||
conditions.push(`severity_text IN (${escaped})`);
|
||||
}
|
||||
|
||||
if (search) {
|
||||
conditions.push(`body ILIKE ${sqlstring.escape(`%${search}%`)}`);
|
||||
}
|
||||
|
||||
if (loggerName) {
|
||||
conditions.push(`logger_name = ${sqlstring.escape(loggerName)}`);
|
||||
}
|
||||
|
||||
if (startDate) {
|
||||
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
|
||||
}
|
||||
|
||||
if (endDate) {
|
||||
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
|
||||
}
|
||||
|
||||
const where = conditions.join(' AND ');
|
||||
|
||||
const rows = await chQuery<IClickhouseLog>(
|
||||
`SELECT
|
||||
id, project_id, device_id, profile_id, session_id,
|
||||
timestamp, severity_number, severity_text, body,
|
||||
trace_id, span_id, trace_flags, logger_name,
|
||||
attributes, resource,
|
||||
sdk_name, sdk_version,
|
||||
country, city, region, os, os_version,
|
||||
browser, browser_version, device, brand, model
|
||||
FROM logs
|
||||
WHERE ${where}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ${take + 1}`,
|
||||
);
|
||||
|
||||
const hasMore = rows.length > take;
|
||||
const data = rows.slice(0, take).map(toServiceLog);
|
||||
const lastItem = data[data.length - 1];
|
||||
|
||||
return {
|
||||
data,
|
||||
meta: {
|
||||
next: hasMore && lastItem ? lastItem.timestamp.toISOString() : null,
|
||||
},
|
||||
};
|
||||
}),
|
||||
|
||||
severityCounts: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
projectId: z.string(),
|
||||
startDate: z.date().optional(),
|
||||
endDate: z.date().optional(),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input }) => {
|
||||
const { projectId, startDate, endDate } = input;
|
||||
|
||||
const conditions: string[] = [
|
||||
`project_id = ${sqlstring.escape(projectId)}`,
|
||||
];
|
||||
|
||||
if (startDate) {
|
||||
conditions.push(`timestamp >= ${sqlstring.escape(startDate.toISOString())}`);
|
||||
}
|
||||
|
||||
if (endDate) {
|
||||
conditions.push(`timestamp <= ${sqlstring.escape(endDate.toISOString())}`);
|
||||
}
|
||||
|
||||
const where = conditions.join(' AND ');
|
||||
|
||||
const rows = await chQuery<{ severity_text: string; count: number }>(
|
||||
`SELECT severity_text, count() AS count
|
||||
FROM logs
|
||||
WHERE ${where}
|
||||
GROUP BY severity_text
|
||||
ORDER BY count DESC`,
|
||||
);
|
||||
|
||||
return rows.reduce<Record<string, number>>((acc, row) => {
|
||||
acc[row.severity_text] = row.count;
|
||||
return acc;
|
||||
}, {});
|
||||
}),
|
||||
});
|
||||
@@ -13,7 +13,7 @@
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"prisma": "^5.1.1",
|
||||
"prisma": "^5.22.0",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,3 +626,4 @@ export type ICreateImport = z.infer<typeof zCreateImport>;
|
||||
export * from './types.insights';
|
||||
export * from './track.validation';
|
||||
export * from './event-blocklist';
|
||||
export * from './log.validation';
|
||||
|
||||
60
packages/validation/src/log.validation.ts
Normal file
60
packages/validation/src/log.validation.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* OTel severity number mapping (subset):
|
||||
* TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21
|
||||
*/
|
||||
export const SEVERITY_TEXT_TO_NUMBER: Record<string, number> = {
|
||||
trace: 1,
|
||||
debug: 5,
|
||||
info: 9,
|
||||
warn: 13,
|
||||
warning: 13,
|
||||
error: 17,
|
||||
fatal: 21,
|
||||
critical: 21,
|
||||
};
|
||||
|
||||
export const zSeverityText = z.enum([
|
||||
'trace',
|
||||
'debug',
|
||||
'info',
|
||||
'warn',
|
||||
'warning',
|
||||
'error',
|
||||
'fatal',
|
||||
'critical',
|
||||
]);
|
||||
|
||||
export type ISeverityText = z.infer<typeof zSeverityText>;
|
||||
|
||||
export const zLogPayload = z.object({
|
||||
/** Log message / body */
|
||||
body: z.string().min(1),
|
||||
/** Severity level as text */
|
||||
severity: zSeverityText.default('info'),
|
||||
/** Optional override for the numeric OTel severity (1-24) */
|
||||
severityNumber: z.number().int().min(1).max(24).optional(),
|
||||
/** ISO 8601 timestamp; defaults to server receive time if omitted */
|
||||
timestamp: z.string().datetime({ offset: true }).optional(),
|
||||
/** Logger name (e.g. "com.example.MyActivity") */
|
||||
loggerName: z.string().optional(),
|
||||
/** W3C trace context */
|
||||
traceId: z.string().optional(),
|
||||
spanId: z.string().optional(),
|
||||
traceFlags: z.number().int().min(0).optional(),
|
||||
/** Log-level key-value attributes */
|
||||
attributes: z.record(z.string(), z.string()).optional(),
|
||||
/** Resource/device attributes (app version, runtime, etc.) */
|
||||
resource: z.record(z.string(), z.string()).optional(),
|
||||
/** Profile/user ID to associate with this log */
|
||||
profileId: z.union([z.string().min(1), z.number()]).optional(),
|
||||
});
|
||||
|
||||
export type ILogPayload = z.infer<typeof zLogPayload>;
|
||||
|
||||
export const zLogBatchPayload = z.object({
|
||||
logs: z.array(zLogPayload).min(1).max(500),
|
||||
});
|
||||
|
||||
export type ILogBatchPayload = z.infer<typeof zLogBatchPayload>;
|
||||
42
patches/buffer-equal-constant-time.patch
Normal file
42
patches/buffer-equal-constant-time.patch
Normal file
@@ -0,0 +1,42 @@
|
||||
diff --git a/.npmignore b/.npmignore
|
||||
deleted file mode 100644
|
||||
index 34e4f5c298de294fa5c1c1769b6489eb047bde9a..0000000000000000000000000000000000000000
|
||||
diff --git a/index.js b/index.js
|
||||
index 5462c1f830bdbe79bf2b1fcfd811cd9799b4dd11..689421d49e83d168981a0c7d1fef59a0b0e56963 100644
|
||||
--- a/index.js
|
||||
+++ b/index.js
|
||||
@@ -3,6 +3,9 @@
|
||||
var Buffer = require('buffer').Buffer; // browserify
|
||||
var SlowBuffer = require('buffer').SlowBuffer;
|
||||
|
||||
+// Handle Node.js v25+ where SlowBuffer was removed
|
||||
+var hasSlowBuffer = !!SlowBuffer;
|
||||
+
|
||||
module.exports = bufferEq;
|
||||
|
||||
function bufferEq(a, b) {
|
||||
@@ -28,14 +31,18 @@ function bufferEq(a, b) {
|
||||
}
|
||||
|
||||
bufferEq.install = function() {
|
||||
- Buffer.prototype.equal = SlowBuffer.prototype.equal = function equal(that) {
|
||||
- return bufferEq(this, that);
|
||||
- };
|
||||
+ Buffer.prototype.equal = function equal(that) {
|
||||
+ return bufferEq(this, that);
|
||||
+ };
|
||||
+ if (hasSlowBuffer) {
|
||||
+ SlowBuffer.prototype.equal = Buffer.prototype.equal;
|
||||
+ }
|
||||
};
|
||||
|
||||
var origBufEqual = Buffer.prototype.equal;
|
||||
-var origSlowBufEqual = SlowBuffer.prototype.equal;
|
||||
bufferEq.restore = function() {
|
||||
- Buffer.prototype.equal = origBufEqual;
|
||||
- SlowBuffer.prototype.equal = origSlowBufEqual;
|
||||
+ Buffer.prototype.equal = origBufEqual;
|
||||
+ if (hasSlowBuffer && SlowBuffer.prototype) {
|
||||
+ delete SlowBuffer.prototype.equal;
|
||||
+ }
|
||||
};
|
||||
28713
pnpm-lock.yaml
generated
28713
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -7,10 +7,10 @@ packages:
|
||||
# Define a catalog of version ranges.
|
||||
catalog:
|
||||
zod: ^3.24.2
|
||||
react: ^19.2.3
|
||||
"@types/react": ^19.2.3
|
||||
"react-dom": ^19.2.3
|
||||
react: ^19.2.4
|
||||
"@types/react": ^19.2.14
|
||||
"react-dom": ^19.2.4
|
||||
"@types/react-dom": ^19.2.3
|
||||
"@types/node": ^24.7.1
|
||||
"@types/node": ^25.5.0
|
||||
typescript: ^5.9.3
|
||||
groupmq: 2.0.0-next.1
|
||||
|
||||
@@ -23,7 +23,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
op-db:
|
||||
image: postgres:14-alpine
|
||||
image: postgres:18.3-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-db-data:/var/lib/postgresql/data
|
||||
@@ -45,7 +45,7 @@ services:
|
||||
# - 5432:5432
|
||||
|
||||
op-kv:
|
||||
image: redis:7.2.5-alpine
|
||||
image: redis:8.6.2-alpine
|
||||
restart: always
|
||||
volumes:
|
||||
- op-kv-data:/data
|
||||
@@ -65,7 +65,7 @@ services:
|
||||
# - 6379:6379
|
||||
|
||||
op-ch:
|
||||
image: clickhouse/clickhouse-server:25.10.2.65
|
||||
image: clickhouse/clickhouse-server:26.3.2.3
|
||||
restart: always
|
||||
volumes:
|
||||
- op-ch-data:/var/lib/clickhouse
|
||||
|
||||
@@ -9,12 +9,12 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"arg": "^5.0.2",
|
||||
"semver": "^7.5.4"
|
||||
"semver": "^7.7.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openpanel/tsconfig": "workspace:*",
|
||||
"@types/node": "catalog:",
|
||||
"@types/semver": "^7.5.4",
|
||||
"@types/semver": "^7.7.1",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"files": ["base.json", "sdk.json", "tsup.config.json"],
|
||||
"files": [
|
||||
"base.json",
|
||||
"sdk.json",
|
||||
"tsup.config.json"
|
||||
],
|
||||
"devDependencies": {
|
||||
"tsup": "^7.2.0"
|
||||
"tsup": "^8.5.1"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user