fix(buffer): chunk inserts

This commit is contained in:
Carl-Gerhard Lindesvärd
2025-01-29 08:47:35 +00:00
parent 0ffca70782
commit 033a6acd43

View File

@@ -222,12 +222,22 @@ export class EventBuffer extends RedisBuffer<BufferType> {
}; };
} }
private getChunks(items: BufferType[], size: number) {
const chunks = [];
for (let i = 0; i < items.length; i += size) {
chunks.push(items.slice(i, i + size));
}
return chunks;
}
protected async insertIntoDB(items: BufferType[]): Promise<void> { protected async insertIntoDB(items: BufferType[]): Promise<void> {
await ch.insert({ for (const chunk of this.getChunks(items, 1500)) {
table: TABLE_NAMES.events, await ch.insert({
values: items, table: TABLE_NAMES.events,
format: 'JSONEachRow', values: chunk,
}); format: 'JSONEachRow',
});
}
} }
public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async ( public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (