zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

test(buffer): disable auto flush on profile

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-09-14 22:36:32 +02:00
parent 7219921a80
commit c674127784
3 changed files with 11 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
apps/worker/src/index.ts
View File

@@ -175,9 +175,7 @@ async function start() {
{ {
jobId: 'flushProfiles', jobId: 'flushProfiles',
repeat: { repeat: {
every: process.env.BATCH_INTERVAL every: 2 * 1000,
? parseInt(process.env.BATCH_INTERVAL, 10)
: 1000 * 10,
}, },
} }
); );
@@ -200,8 +198,7 @@ async function start() {
const repeatableJobs = await cronQueue.getRepeatableJobs(); const repeatableJobs = await cronQueue.getRepeatableJobs();
console.log('Repeatable jobs:'); logger.info('Repeatable jobs', { repeatableJobs });
console.log(repeatableJobs);
await createInitialSalts(); await createInitialSalts();
} }

10
packages/db/src/buffers/buffer.ts
View File

@@ -42,6 +42,7 @@ export abstract class RedisBuffer<T> {
public table: string; public table: string;
public batchSize?: number; public batchSize?: number;
public logger: ReturnType<typeof createLogger>; public logger: ReturnType<typeof createLogger>;
public disableAutoFlush?: boolean;
// abstract methods // abstract methods
public abstract onInsert?: OnInsert<T>; public abstract onInsert?: OnInsert<T>;
@@ -50,9 +51,14 @@ export abstract class RedisBuffer<T> {
public abstract find: Find<T, unknown>; public abstract find: Find<T, unknown>;
public abstract findMany: FindMany<T, unknown>; public abstract findMany: FindMany<T, unknown>;
constructor(options: { table: string; batchSize?: number }) { constructor(options: {
table: string;
batchSize?: number;
disableAutoFlush?: boolean;
}) {
this.table = options.table; this.table = options.table;
this.batchSize = options.batchSize; this.batchSize = options.batchSize;
this.disableAutoFlush = options.disableAutoFlush;
this.logger = createLogger({ name: `buffer` }).child({ this.logger = createLogger({ name: `buffer` }).child({
table: this.table, table: this.table,
}); });
@@ -75,7 +81,7 @@ export abstract class RedisBuffer<T> {
`Inserted item into buffer ${this.table}. Current length: ${length}` `Inserted item into buffer ${this.table}. Current length: ${length}`
); );
if (this.batchSize && length >= this.batchSize) { if (!this.disableAutoFlush && this.batchSize && length >= this.batchSize) {
this.logger.info( this.logger.info(
`Buffer ${this.table} reached batch size (${this.batchSize}). Flushing...` `Buffer ${this.table} reached batch size (${this.batchSize}). Flushing...`
); );

1
packages/db/src/buffers/profile-buffer.ts
View File

@@ -24,6 +24,7 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {
super({ super({
table: TABLE_NAMES.profiles, table: TABLE_NAMES.profiles,
batchSize: 100, batchSize: 100,
disableAutoFlush: true,
}); });
} }