zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: overall perf improvements

Browse Source 
* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-15 22:13:59 +01:00
committed by GitHub
parent 38cc53890a
commit da59622dce
66 changed files with 5042 additions and 3860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
apps/worker/src/metrics.ts
View File

@@ -2,23 +2,32 @@ import client from 'prom-client';
import {
botBuffer,
db,
eventBuffer,
profileBuffer,
sessionBuffer,
} from '@openpanel/db';
import { cronQueue, eventsGroupQueue, sessionsQueue } from '@openpanel/queue';
import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
const Registry = client.Registry;
export const register = new Registry();
const queues = [sessionsQueue, cronQueue, eventsGroupQueue];
const queues = [sessionsQueue, cronQueue, ...eventsGroupQueues];
// Histogram to track job processing time for eventsGroupQueues
export const eventsGroupJobDuration = new client.Histogram({
name: 'job_duration_ms',
help: 'Duration of job processing (in ms)',
labelNames: ['name', 'status'],
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
});
register.registerMetric(eventsGroupJobDuration);
queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_active_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
help: 'Active count',
async collect() {
const metric = await queue.getActiveCount();
@@ -29,7 +38,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_delayed_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
help: 'Delayed count',
async collect() {
const metric = await queue.getDelayedCount();
@@ -40,7 +49,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_failed_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
help: 'Failed count',
async collect() {
const metric = await queue.getFailedCount();
@@ -51,7 +60,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_completed_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
help: 'Completed count',
async collect() {
const metric = await queue.getCompletedCount();
@@ -62,7 +71,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_waiting_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
help: 'Waiting count',
async collect() {
const metric = await queue.getWaitingCount();