zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: use groupmq instead of bullmq for incoming events (#206)

Browse Source 
* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-04 21:07:55 +02:00
committed by GitHub
parent ca4a880acd
commit 0b4fcbad69
23 changed files with 1292 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
packages/db/src/buffers/session-buffer.ts
View File

@@ -1,4 +1,4 @@
import { type Redis, getRedisCache, runEvery } from '@openpanel/redis';
import { type Redis, getRedisCache } from '@openpanel/redis';
import { toDots } from '@openpanel/common';
import { getSafeJson } from '@openpanel/json';
@@ -61,7 +61,7 @@ export class SessionBuffer extends BaseBuffer {
const duration =
new Date(newSession.ended_at).getTime() -
new Date(newSession.created_at).getTime();
if (duration > 0) {
if (duration >= 0) {
newSession.duration = duration;
} else {
this.logger.warn('Session duration is negative', {
@@ -174,10 +174,12 @@ export class SessionBuffer extends BaseBuffer {
for (const session of sessions) {
multi.rpush(this.redisKey, JSON.stringify(session));
}
// Increment counter by number of sessions added
multi.incrby(this.bufferCounterKey, sessions.length);
await multi.exec();
// Check buffer length
const bufferLength = await this.redis.llen(this.redisKey);
// Check buffer length using counter
const bufferLength = await this.getBufferSize();
if (bufferLength >= this.batchSize) {
await this.tryFlush();
@@ -216,8 +218,12 @@ export class SessionBuffer extends BaseBuffer {
});
}
// Only remove events after successful insert
await this.redis.ltrim(this.redisKey, events.length, -1);
// Only remove events after successful insert and update counter
const multi = this.redis.multi();
multi
.ltrim(this.redisKey, events.length, -1)
.decrby(this.bufferCounterKey, events.length);
await multi.exec();
this.logger.info('Processed sessions', {
count: events.length,
@@ -228,6 +234,6 @@ export class SessionBuffer extends BaseBuffer {
}
async getBufferSize() {
return getRedisCache().llen(this.redisKey);
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
}
}