From 033a6acd4396bdc0a7fb2a48403ed6b2e78b75fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl-Gerhard=20Lindesva=CC=88rd?= <lindesvard@gmail.com>
Date: Wed, 29 Jan 2025 08:47:35 +0000
Subject: [PATCH] fix(buffer): chunk inserts

---
 packages/db/src/buffers/event-buffer.ts | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.ts
index 76f2d288..d9e5346e 100644
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/db/src/buffers/event-buffer.ts
@@ -222,12 +222,22 @@ export class EventBuffer extends RedisBuffer<BufferType> {
     };
   }
 
+  private getChunks(items: BufferType[], size: number) {
+    const chunks = [];
+    for (let i = 0; i < items.length; i += size) {
+      chunks.push(items.slice(i, i + size));
+    }
+    return chunks;
+  }
+
   protected async insertIntoDB(items: BufferType[]): Promise<void> {
-    await ch.insert({
-      table: TABLE_NAMES.events,
-      values: items,
-      format: 'JSONEachRow',
-    });
+    for (const chunk of this.getChunks(items, 1500)) {
+      await ch.insert({
+        table: TABLE_NAMES.events,
+        values: chunk,
+        format: 'JSONEachRow',
+      });
+    }
   }
 
   public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (