fix(buffer): chunk inserts

This commit is contained in:
Carl-Gerhard Lindesvärd
2025-01-29 08:47:35 +00:00
parent 0ffca70782
commit 033a6acd43

View File

@@ -222,13 +222,23 @@ export class EventBuffer extends RedisBuffer<BufferType> {
}; };
} }
private getChunks(items: BufferType[], size: number) {
const chunks = [];
for (let i = 0; i < items.length; i += size) {
chunks.push(items.slice(i, i + size));
}
return chunks;
}
protected async insertIntoDB(items: BufferType[]): Promise<void> { protected async insertIntoDB(items: BufferType[]): Promise<void> {
for (const chunk of this.getChunks(items, 1500)) {
await ch.insert({ await ch.insert({
table: TABLE_NAMES.events, table: TABLE_NAMES.events,
values: items, values: chunk,
format: 'JSONEachRow', format: 'JSONEachRow',
}); });
} }
}
public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async ( public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (
callback, callback,