perf: optimize event buffer
This commit is contained in:
Carl-Gerhard Lindesvärd
@@ -25,8 +25,21 @@ import { BaseBuffer } from './base-buffer';
|
||||
* - Retrieve the last screen_view (don't modify it)
|
||||
* - Push both screen_view and session_end to buffer
|
||||
* 4. Flush: Simply process all events from the list buffer
|
||||
*
|
||||
* Optimizations:
|
||||
* - Micro-batching: Events are buffered locally and flushed every 10ms to reduce Redis round-trips
|
||||
* - Batched publishes: All PUBLISH commands are included in the multi pipeline
|
||||
* - Simplified active visitor tracking: Only uses ZADD (removed redundant heartbeat SET)
|
||||
*/
|
||||
|
||||
// Pending event for local buffer
|
||||
interface PendingEvent {
|
||||
event: IClickhouseEvent;
|
||||
eventJson: string;
|
||||
eventWithTimestamp?: string;
|
||||
type: 'regular' | 'screen_view' | 'session_end';
|
||||
}
|
||||
|
||||
export class EventBuffer extends BaseBuffer {
|
||||
// Configurable limits
|
||||
private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE
|
||||
@@ -36,6 +49,27 @@ export class EventBuffer extends BaseBuffer {
|
||||
? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)
|
||||
: 1000;
|
||||
|
||||
// Micro-batching configuration
|
||||
private microBatchIntervalMs = process.env.EVENT_BUFFER_MICRO_BATCH_MS
|
||||
? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_MS, 10)
|
||||
: 10; // Flush every 10ms by default
|
||||
private microBatchMaxSize = process.env.EVENT_BUFFER_MICRO_BATCH_SIZE
|
||||
? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_SIZE, 10)
|
||||
: 100; // Or when we hit 100 events
|
||||
|
||||
// Local event buffer for micro-batching
|
||||
private pendingEvents: PendingEvent[] = [];
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private isFlushing = false;
|
||||
|
||||
// Throttled publish configuration
|
||||
private publishThrottleMs = process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS
|
||||
? Number.parseInt(process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS, 10)
|
||||
: 1000; // Publish at most once per second
|
||||
private lastPublishTime = 0;
|
||||
private pendingPublishEvent: IClickhouseEvent | null = null;
|
||||
private publishTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
private activeVisitorsExpiration = 60 * 5; // 5 minutes
|
||||
|
||||
// LIST - Stores all events ready to be flushed
|
||||
@@ -190,98 +224,228 @@ return added
|
||||
}
|
||||
|
||||
bulkAdd(events: IClickhouseEvent[]) {
|
||||
const redis = getRedisCache();
|
||||
const multi = redis.multi();
|
||||
// Add all events to local buffer - they will be flushed together
|
||||
for (const event of events) {
|
||||
this.add(event, multi);
|
||||
this.add(event);
|
||||
}
|
||||
return multi.exec();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an event into Redis buffer.
|
||||
* Add an event into the local buffer for micro-batching.
|
||||
*
|
||||
* Events are buffered locally and flushed to Redis every microBatchIntervalMs
|
||||
* or when microBatchMaxSize is reached. This dramatically reduces Redis round-trips.
|
||||
*
|
||||
* Logic:
|
||||
* - screen_view: Store as "last" for session, flush previous if exists
|
||||
* - session_end: Flush last screen_view + session_end
|
||||
* - Other events: Add directly to queue
|
||||
*/
|
||||
async add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
|
||||
add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
|
||||
const eventJson = JSON.stringify(event);
|
||||
|
||||
// Determine event type and prepare data
|
||||
let type: PendingEvent['type'] = 'regular';
|
||||
let eventWithTimestamp: string | undefined;
|
||||
|
||||
if (event.session_id && event.name === 'screen_view') {
|
||||
type = 'screen_view';
|
||||
const timestamp = new Date(event.created_at || Date.now()).getTime();
|
||||
eventWithTimestamp = JSON.stringify({
|
||||
event: event,
|
||||
ts: timestamp,
|
||||
});
|
||||
} else if (event.session_id && event.name === 'session_end') {
|
||||
type = 'session_end';
|
||||
}
|
||||
|
||||
const pendingEvent: PendingEvent = {
|
||||
event,
|
||||
eventJson,
|
||||
eventWithTimestamp,
|
||||
type,
|
||||
};
|
||||
|
||||
// If a multi was provided (legacy bulkAdd pattern), add directly without batching
|
||||
if (_multi) {
|
||||
this.addToMulti(_multi, pendingEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
// Add to local buffer for micro-batching
|
||||
this.pendingEvents.push(pendingEvent);
|
||||
|
||||
// Check if we should flush immediately due to size
|
||||
if (this.pendingEvents.length >= this.microBatchMaxSize) {
|
||||
this.flushLocalBuffer();
|
||||
return;
|
||||
}
|
||||
|
||||
// Schedule flush if not already scheduled
|
||||
if (!this.flushTimer) {
|
||||
this.flushTimer = setTimeout(() => {
|
||||
this.flushTimer = null;
|
||||
this.flushLocalBuffer();
|
||||
}, this.microBatchIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a single pending event to a multi pipeline.
|
||||
* Used both for legacy _multi pattern and during batch flush.
|
||||
*/
|
||||
private addToMulti(multi: ReturnType<Redis['multi']>, pending: PendingEvent) {
|
||||
const { event, eventJson, eventWithTimestamp, type } = pending;
|
||||
|
||||
if (type === 'screen_view' && event.session_id) {
|
||||
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
|
||||
const profileKey = event.profile_id
|
||||
? this.getLastScreenViewKeyByProfile(event.project_id, event.profile_id)
|
||||
: '';
|
||||
|
||||
this.evalScript(
|
||||
multi,
|
||||
'addScreenView',
|
||||
this.addScreenViewScript,
|
||||
4,
|
||||
sessionKey,
|
||||
profileKey,
|
||||
this.queueKey,
|
||||
this.bufferCounterKey,
|
||||
eventWithTimestamp!,
|
||||
'3600',
|
||||
);
|
||||
} else if (type === 'session_end' && event.session_id) {
|
||||
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
|
||||
const profileKey = event.profile_id
|
||||
? this.getLastScreenViewKeyByProfile(event.project_id, event.profile_id)
|
||||
: '';
|
||||
|
||||
this.evalScript(
|
||||
multi,
|
||||
'addSessionEnd',
|
||||
this.addSessionEndScript,
|
||||
4,
|
||||
sessionKey,
|
||||
profileKey,
|
||||
this.queueKey,
|
||||
this.bufferCounterKey,
|
||||
eventJson,
|
||||
);
|
||||
} else {
|
||||
// Regular events go directly to queue
|
||||
multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
|
||||
}
|
||||
|
||||
// Active visitor tracking (simplified - only ZADD, no redundant SET)
|
||||
if (event.profile_id) {
|
||||
this.incrementActiveVisitorCount(
|
||||
multi,
|
||||
event.project_id,
|
||||
event.profile_id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force flush all pending events from local buffer to Redis immediately.
|
||||
* Useful for testing or when you need to ensure all events are persisted.
|
||||
*/
|
||||
public async flush() {
|
||||
// Clear any pending timer
|
||||
if (this.flushTimer) {
|
||||
clearTimeout(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
await this.flushLocalBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush all pending events from local buffer to Redis in a single pipeline.
|
||||
* This is the core optimization - batching many events into one round-trip.
|
||||
*/
|
||||
private async flushLocalBuffer() {
|
||||
if (this.isFlushing || this.pendingEvents.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isFlushing = true;
|
||||
|
||||
// Grab current pending events and clear buffer
|
||||
const eventsToFlush = this.pendingEvents;
|
||||
this.pendingEvents = [];
|
||||
|
||||
try {
|
||||
const redis = getRedisCache();
|
||||
const eventJson = JSON.stringify(event);
|
||||
const multi = _multi || redis.multi();
|
||||
const multi = redis.multi();
|
||||
|
||||
if (event.session_id && event.name === 'screen_view') {
|
||||
// Handle screen_view
|
||||
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
|
||||
const profileKey = event.profile_id
|
||||
? this.getLastScreenViewKeyByProfile(
|
||||
event.project_id,
|
||||
event.profile_id,
|
||||
)
|
||||
: '';
|
||||
const timestamp = new Date(event.created_at || Date.now()).getTime();
|
||||
|
||||
// Combine event and timestamp into single JSON for atomic operations
|
||||
const eventWithTimestamp = JSON.stringify({
|
||||
event: event,
|
||||
ts: timestamp,
|
||||
});
|
||||
|
||||
this.evalScript(
|
||||
multi,
|
||||
'addScreenView',
|
||||
this.addScreenViewScript,
|
||||
4,
|
||||
sessionKey,
|
||||
profileKey,
|
||||
this.queueKey,
|
||||
this.bufferCounterKey,
|
||||
eventWithTimestamp,
|
||||
'3600', // 1 hour TTL
|
||||
);
|
||||
} else if (event.session_id && event.name === 'session_end') {
|
||||
// Handle session_end
|
||||
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
|
||||
const profileKey = event.profile_id
|
||||
? this.getLastScreenViewKeyByProfile(
|
||||
event.project_id,
|
||||
event.profile_id,
|
||||
)
|
||||
: '';
|
||||
|
||||
this.evalScript(
|
||||
multi,
|
||||
'addSessionEnd',
|
||||
this.addSessionEndScript,
|
||||
4,
|
||||
sessionKey,
|
||||
profileKey,
|
||||
this.queueKey,
|
||||
this.bufferCounterKey,
|
||||
eventJson,
|
||||
);
|
||||
} else {
|
||||
// All other events go directly to queue
|
||||
multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
|
||||
// Add all events to the pipeline
|
||||
for (const pending of eventsToFlush) {
|
||||
this.addToMulti(multi, pending);
|
||||
}
|
||||
|
||||
if (event.profile_id) {
|
||||
this.incrementActiveVisitorCount(
|
||||
multi,
|
||||
event.project_id,
|
||||
event.profile_id,
|
||||
);
|
||||
}
|
||||
await multi.exec();
|
||||
|
||||
if (!_multi) {
|
||||
await multi.exec();
|
||||
// Throttled publish - just signal that events were received
|
||||
// Store the last event for publishing (we only need one to signal activity)
|
||||
const lastEvent = eventsToFlush[eventsToFlush.length - 1];
|
||||
if (lastEvent) {
|
||||
this.scheduleThrottledPublish(lastEvent.event);
|
||||
}
|
||||
|
||||
await publishEvent('events', 'received', transformEvent(event));
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to add event to Redis buffer', { error });
|
||||
this.logger.error('Failed to flush local buffer to Redis', {
|
||||
error,
|
||||
eventCount: eventsToFlush.length,
|
||||
});
|
||||
} finally {
|
||||
this.isFlushing = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttled publish - publishes at most once per publishThrottleMs.
|
||||
* Instead of publishing every event, we just signal that events were received.
|
||||
* This reduces pub/sub load from 3000/s to 1/s.
|
||||
*/
|
||||
private scheduleThrottledPublish(event: IClickhouseEvent) {
|
||||
// Always keep the latest event
|
||||
this.pendingPublishEvent = event;
|
||||
|
||||
const now = Date.now();
|
||||
const timeSinceLastPublish = now - this.lastPublishTime;
|
||||
|
||||
// If enough time has passed, publish immediately
|
||||
if (timeSinceLastPublish >= this.publishThrottleMs) {
|
||||
this.executeThrottledPublish();
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, schedule a publish if not already scheduled
|
||||
if (!this.publishTimer) {
|
||||
const delay = this.publishThrottleMs - timeSinceLastPublish;
|
||||
this.publishTimer = setTimeout(() => {
|
||||
this.publishTimer = null;
|
||||
this.executeThrottledPublish();
|
||||
}, delay);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the throttled publish with the latest pending event.
|
||||
*/
|
||||
private executeThrottledPublish() {
|
||||
if (!this.pendingPublishEvent) {
|
||||
return;
|
||||
}
|
||||
|
||||
const event = this.pendingPublishEvent;
|
||||
this.pendingPublishEvent = null;
|
||||
this.lastPublishTime = Date.now();
|
||||
|
||||
// Fire-and-forget publish (no multi = returns Promise)
|
||||
const result = publishEvent('events', 'received', transformEvent(event));
|
||||
if (result instanceof Promise) {
|
||||
result.catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,18 +604,22 @@ return added
|
||||
});
|
||||
}
|
||||
|
||||
private async incrementActiveVisitorCount(
|
||||
/**
|
||||
* Track active visitors using ZADD only.
|
||||
*
|
||||
* Optimization: Removed redundant heartbeat SET key.
|
||||
* The ZADD score (timestamp) already tracks when a visitor was last seen.
|
||||
* We use ZRANGEBYSCORE in getActiveVisitorCount to filter active visitors.
|
||||
*/
|
||||
private incrementActiveVisitorCount(
|
||||
multi: ReturnType<Redis['multi']>,
|
||||
projectId: string,
|
||||
profileId: string,
|
||||
) {
|
||||
// Track active visitors and emit expiry events when inactive for TTL
|
||||
const now = Date.now();
|
||||
const zsetKey = `live:visitors:${projectId}`;
|
||||
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
|
||||
return multi
|
||||
.zadd(zsetKey, now, profileId)
|
||||
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
|
||||
// Only ZADD - the score is the timestamp, no need for separate heartbeat key
|
||||
return multi.zadd(zsetKey, now, profileId);
|
||||
}
|
||||
|
||||
public async getActiveVisitorCount(projectId: string): Promise<number> {
|
||||
|
||||
Reference in New Issue
Block a user