zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: optimize event buffer (#278)

Browse Source 
* fix: how we fetch profiles in the buffer

* perf: optimize event buffer

* remove unused file

* fix

* wip

* wip: try groupmq 2

* try simplified event buffer with duration calculation on the fly instead
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-16 13:29:40 +01:00
committed by GitHub
parent 4736f8509d
commit 4483e464d1
46 changed files with 887 additions and 1841 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
apps/worker/src/metrics.ts
View File

@@ -1,5 +1,3 @@
import client from 'prom-client';
import {
botBuffer,
eventBuffer,
@@ -8,6 +6,7 @@ import {
sessionBuffer,
} from '@openpanel/db';
import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
import client from 'prom-client';
const Registry = client.Registry;
@@ -20,7 +19,7 @@ export const eventsGroupJobDuration = new client.Histogram({
name: 'job_duration_ms',
help: 'Duration of job processing (in ms)',
labelNames: ['name', 'status'],
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10_000, 30_000], // 10ms to 30s
});
register.registerMetric(eventsGroupJobDuration);
@@ -28,57 +27,61 @@ register.registerMetric(eventsGroupJobDuration);
queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
name: `${queue.name.replace(/[{}]/g, '')}_active_count`,
help: 'Active count',
async collect() {
const metric = await queue.getActiveCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_delayed_count`,
help: 'Delayed count',
async collect() {
const metric = await queue.getDelayedCount();
this.set(metric);
if ('getDelayedCount' in queue) {
const metric = await queue.getDelayedCount();
this.set(metric);
} else {
this.set(0);
}
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_failed_count`,
help: 'Failed count',
async collect() {
const metric = await queue.getFailedCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
name: `${queue.name.replace(/[{}]/g, '')}_completed_count`,
help: 'Completed count',
async collect() {
const metric = await queue.getCompletedCount();
this.set(metric);
},
}),
})
);
register.registerMetric(
new client.Gauge({
name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
name: `${queue.name.replace(/[{}]/g, '')}_waiting_count`,
help: 'Waiting count',
async collect() {
const metric = await queue.getWaitingCount();
this.set(metric);
},
}),
})
);
});
@@ -90,7 +93,7 @@ register.registerMetric(
const metric = await eventBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -101,7 +104,7 @@ register.registerMetric(
const metric = await profileBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -112,7 +115,7 @@ register.registerMetric(
const metric = await botBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -123,7 +126,7 @@ register.registerMetric(
const metric = await sessionBuffer.getBufferSize();
this.set(metric);
},
}),
})
);
register.registerMetric(
@@ -134,5 +137,5 @@ register.registerMetric(
const metric = await replayBuffer.getBufferSize();
this.set(metric);
},
}),
})
);