feat: use groupmq instead of bullmq for incoming events (#206)

* wip * wip working group queue * wip * wip * wip * fix: groupmq package (tests failed) * minor fixes * fix: zero is fine for duration * add logger * fix: make buffers more lightweight * bump groupmq * new buffers and bump groupmq * fix: buffers based on comments * fix: use profileId as groupId if exists * bump groupmq * add concurrency env for only events
2025-10-04 21:07:55 +02:00
parent ca4a880acd
commit 0b4fcbad69
23 changed files with 1292 additions and 354 deletions
--- a/packages/db/src/buffers/bot-buffer.ts
+++ b/packages/db/src/buffers/bot-buffer.ts
@@ -0,0 +1,85 @@
+import { type Redis, getRedisCache } from '@openpanel/redis';
+
+import { getSafeJson } from '@openpanel/json';
+import { TABLE_NAMES, ch } from '../clickhouse/client';
+import type { IClickhouseBotEvent } from '../services/event.service';
+import { BaseBuffer } from './base-buffer';
+
+export class BotBuffer extends BaseBuffer {
+  private batchSize = process.env.BOT_BUFFER_BATCH_SIZE
+    ? Number.parseInt(process.env.BOT_BUFFER_BATCH_SIZE, 10)
+    : 1000;
+
+  private readonly redisKey = 'bot-events-buffer';
+  private redis: Redis;
+  constructor() {
+    super({
+      name: 'bot',
+      onFlush: async () => {
+        await this.processBuffer();
+      },
+    });
+    this.redis = getRedisCache();
+  }
+
+  async add(event: IClickhouseBotEvent) {
+    try {
+      // Add event and increment counter atomically
+      await this.redis
+        .multi()
+        .rpush(this.redisKey, JSON.stringify(event))
+        .incr(this.bufferCounterKey)
+        .exec();
+
+      // Check buffer length using counter (fallback to LLEN if missing)
+      const bufferLength = await this.getBufferSize();
+
+      if (bufferLength >= this.batchSize) {
+        await this.tryFlush();
+      }
+    } catch (error) {
+      this.logger.error('Failed to add bot event', { error });
+    }
+  }
+
+  async processBuffer() {
+    try {
+      // Get events from the start without removing them
+      const events = await this.redis.lrange(
+        this.redisKey,
+        0,
+        this.batchSize - 1,
+      );
+
+      if (events.length === 0) return;
+
+      const parsedEvents = events.map((e) =>
+        getSafeJson<IClickhouseBotEvent>(e),
+      );
+
+      // Insert to ClickHouse
+      await ch.insert({
+        table: TABLE_NAMES.events_bots,
+        values: parsedEvents,
+        format: 'JSONEachRow',
+      });
+
+      // Only remove events after successful insert and update counter
+      await this.redis
+        .multi()
+        .ltrim(this.redisKey, events.length, -1)
+        .decrby(this.bufferCounterKey, events.length)
+        .exec();
+
+      this.logger.info('Processed bot events', {
+        count: events.length,
+      });
+    } catch (error) {
+      this.logger.error('Failed to process buffer', { error });
+    }
+  }
+
+  async getBufferSize() {
+    return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
+  }
+}