fix: bump groupmq, fix stalled sessions in buffers, improve heavy event count

This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-06 21:22:56 +02:00
parent b3e06e985d
commit af580333b4
8 changed files with 434 additions and 90 deletions

View File

@@ -39,7 +39,7 @@ export class BaseBuffer {
const key = this.bufferCounterKey;
try {
await runEvery({
interval: 60 * 15,
interval: 60 * 60,
key: `${this.name}-buffer:resync`,
fn: async () => {
try {

View File

@@ -206,7 +206,7 @@ describe('EventBuffer with real Redis', () => {
expect(await eventBuffer.getBufferSize()).toBe(0);
});
it('adds session to ready set at 2 events and removes after processing', async () => {
it('adds session to ready set at 2 events and removes it when < 2 events remain', async () => {
const s = 'session_ready';
const e1 = {
project_id: 'p3',
@@ -235,9 +235,18 @@ describe('EventBuffer with real Redis', () => {
.mockResolvedValueOnce(undefined as any);
await eventBuffer.processBuffer();
// After processing with one pending left, session should be removed from ready set
// After processing with one pending left, session should be REMOVED from ready set
// It will be re-added when the next event arrives
expect(await redis.zscore('event_buffer:ready_sessions', s)).toBeNull();
expect(insertSpy).toHaveBeenCalled();
// But the session and its data should still exist
const sessionKey = `event_buffer:session:${s}`;
const remaining = await redis.lrange(sessionKey, 0, -1);
expect(remaining.length).toBe(1); // One pending event
expect(
await redis.zscore('event_buffer:sessions_sorted', s),
).not.toBeNull(); // Still in sorted set
});
it('sets last screen_view key and clears it on session_end', async () => {
@@ -427,16 +436,16 @@ describe('EventBuffer with real Redis', () => {
await eb.processBuffer();
// Only consider eval calls for batchUpdateSessionsScript (2 keys, second is total_count)
// Only consider eval calls for batchUpdateSessionsScript (3 keys now: ready, sorted, counter)
const batchEvalCalls = evalSpy.mock.calls.filter(
(call) => call[1] === 2 && call[3] === 'event_buffer:total_count',
(call) => call[1] === 3 && call[4] === 'event_buffer:total_count',
);
const expectedCalls = Math.ceil(numSessions / 3);
expect(batchEvalCalls.length).toBeGreaterThanOrEqual(expectedCalls);
function countSessionsInEvalCall(args: any[]): number {
let idx = 4; // ARGV starts after: script, numKeys, key1, key2
let idx = 5; // ARGV starts after: script, numKeys, key1, key2, key3
let count = 0;
while (idx < args.length) {
if (idx + 3 >= args.length) break;
@@ -448,9 +457,10 @@ describe('EventBuffer with real Redis', () => {
}
for (const call of batchEvalCalls) {
expect(call[1]).toBe(2);
expect(call[1]).toBe(3);
expect(call[2]).toBe('event_buffer:ready_sessions');
expect(call[3]).toBe('event_buffer:total_count');
expect(call[3]).toBe('event_buffer:sessions_sorted');
expect(call[4]).toBe('event_buffer:total_count');
const sessionsInThisCall = countSessionsInEvalCall(call.slice(0));
expect(sessionsInThisCall).toBeLessThanOrEqual(3);
@@ -500,4 +510,260 @@ describe('EventBuffer with real Redis', () => {
insertSpy.mockRestore();
});
it('flushes ALL screen_views when session_end arrives (no pending events)', async () => {
const t0 = Date.now();
const s = 'session_multi_end';
const view1 = {
project_id: 'p10',
profile_id: 'u10',
session_id: s,
name: 'screen_view',
created_at: new Date(t0).toISOString(),
} as any;
const view2 = {
...view1,
created_at: new Date(t0 + 1000).toISOString(),
} as any;
const view3 = {
...view1,
created_at: new Date(t0 + 2000).toISOString(),
} as any;
const end = {
...view1,
name: 'session_end',
created_at: new Date(t0 + 3000).toISOString(),
} as any;
const eb = new EventBuffer();
await eb.add(view1);
await eb.add(view2);
await eb.add(view3);
await eb.add(end);
const insertSpy = vi
.spyOn(ch, 'insert')
.mockResolvedValueOnce(undefined as any);
await eb.processBuffer();
// All 4 events should be flushed (3 screen_views + session_end)
expect(insertSpy).toHaveBeenCalledWith({
format: 'JSONEachRow',
table: 'events',
values: [view1, view2, view3, end],
});
// Session should be completely empty and removed
const sessionKey = `event_buffer:session:${s}`;
const remaining = await redis.lrange(sessionKey, 0, -1);
expect(remaining.length).toBe(0);
// Session should be removed from both sorted sets
expect(await redis.zscore('event_buffer:sessions_sorted', s)).toBeNull();
expect(await redis.zscore('event_buffer:ready_sessions', s)).toBeNull();
insertSpy.mockRestore();
});
it('re-adds session to ready_sessions when new event arrives after processing', async () => {
const t0 = Date.now();
const s = 'session_continued';
const view1 = {
project_id: 'p11',
profile_id: 'u11',
session_id: s,
name: 'screen_view',
created_at: new Date(t0).toISOString(),
} as any;
const view2 = {
...view1,
created_at: new Date(t0 + 1000).toISOString(),
} as any;
const eb = new EventBuffer();
await eb.add(view1);
await eb.add(view2);
const insertSpy = vi
.spyOn(ch, 'insert')
.mockResolvedValue(undefined as any);
// First processing: flush view1, keep view2 pending
await eb.processBuffer();
expect(insertSpy).toHaveBeenCalledWith({
format: 'JSONEachRow',
table: 'events',
values: [{ ...view1, duration: 1000 }],
});
// Session should be REMOVED from ready_sessions (only 1 event left)
expect(await redis.zscore('event_buffer:ready_sessions', s)).toBeNull();
// Add a third screen_view - this should re-add to ready_sessions
const view3 = {
...view1,
created_at: new Date(t0 + 2000).toISOString(),
} as any;
await eb.add(view3);
// NOW it should be back in ready_sessions (2 events again)
expect(await redis.zscore('event_buffer:ready_sessions', s)).not.toBeNull();
insertSpy.mockClear();
// Second processing: should process view2 (now has duration), keep view3 pending
await eb.processBuffer();
expect(insertSpy).toHaveBeenCalledWith({
format: 'JSONEachRow',
table: 'events',
values: [{ ...view2, duration: 1000 }],
});
// Session should be REMOVED again (only 1 event left)
expect(await redis.zscore('event_buffer:ready_sessions', s)).toBeNull();
const sessionKey = `event_buffer:session:${s}`;
const remaining = await redis.lrange(sessionKey, 0, -1);
expect(remaining.length).toBe(1);
expect(JSON.parse(remaining[0]!)).toMatchObject({
session_id: s,
created_at: view3.created_at,
});
insertSpy.mockRestore();
});
it('removes session from ready_sessions only when completely empty', async () => {
const t0 = Date.now();
const s = 'session_complete';
const view = {
project_id: 'p12',
profile_id: 'u12',
session_id: s,
name: 'screen_view',
created_at: new Date(t0).toISOString(),
} as any;
const end = {
...view,
name: 'session_end',
created_at: new Date(t0 + 1000).toISOString(),
} as any;
const eb = new EventBuffer();
await eb.add(view);
await eb.add(end);
const insertSpy = vi
.spyOn(ch, 'insert')
.mockResolvedValueOnce(undefined as any);
await eb.processBuffer();
// Both events flushed, session empty
expect(insertSpy).toHaveBeenCalledWith({
format: 'JSONEachRow',
table: 'events',
values: [view, end],
});
// NOW it should be removed from ready_sessions (because it's empty)
expect(await redis.zscore('event_buffer:ready_sessions', s)).toBeNull();
expect(await redis.zscore('event_buffer:sessions_sorted', s)).toBeNull();
insertSpy.mockRestore();
});
it('getBufferSizeHeavy correctly counts events across many sessions in batches', async () => {
const eb = new EventBuffer();
const numSessions = 250; // More than batch size (100) to test batching
const eventsPerSession = 3;
const numRegularEvents = 50;
// Add session events (3 events per session)
for (let i = 0; i < numSessions; i++) {
const sessionId = `batch_session_${i}`;
for (let j = 0; j < eventsPerSession; j++) {
await eb.add({
project_id: 'p_batch',
profile_id: `u_${i}`,
session_id: sessionId,
name: 'screen_view',
created_at: new Date(Date.now() + i * 100 + j * 10).toISOString(),
} as any);
}
}
// Add regular queue events
for (let i = 0; i < numRegularEvents; i++) {
await eb.add({
project_id: 'p_batch',
name: 'custom_event',
created_at: new Date().toISOString(),
} as any);
}
// Get buffer size using heavy method
const bufferSize = await eb.getBufferSizeHeavy();
// Should count all events: (250 sessions × 3 events) + 50 regular events
const expectedSize = numSessions * eventsPerSession + numRegularEvents;
expect(bufferSize).toBe(expectedSize);
// Verify sessions are properly tracked
const sessionCount = await redis.zcard('event_buffer:sessions_sorted');
expect(sessionCount).toBe(numSessions);
const regularQueueCount = await redis.llen('event_buffer:regular_queue');
expect(regularQueueCount).toBe(numRegularEvents);
});
it('getBufferSizeHeavy handles empty buffer correctly', async () => {
const eb = new EventBuffer();
const bufferSize = await eb.getBufferSizeHeavy();
expect(bufferSize).toBe(0);
});
it('getBufferSizeHeavy handles only regular queue events', async () => {
const eb = new EventBuffer();
const numEvents = 10;
for (let i = 0; i < numEvents; i++) {
await eb.add({
project_id: 'p_regular',
name: 'custom_event',
created_at: new Date().toISOString(),
} as any);
}
const bufferSize = await eb.getBufferSizeHeavy();
expect(bufferSize).toBe(numEvents);
});
it('getBufferSizeHeavy handles only session events', async () => {
const eb = new EventBuffer();
const numSessions = 5;
const eventsPerSession = 2;
for (let i = 0; i < numSessions; i++) {
for (let j = 0; j < eventsPerSession; j++) {
await eb.add({
project_id: 'p_sessions',
profile_id: `u_${i}`,
session_id: `session_${i}`,
name: 'screen_view',
created_at: new Date(Date.now() + i * 100 + j * 10).toISOString(),
} as any);
}
}
const bufferSize = await eb.getBufferSizeHeavy();
expect(bufferSize).toBe(numSessions * eventsPerSession);
});
});

View File

@@ -100,6 +100,7 @@ export class EventBuffer extends BaseBuffer {
private readonly processReadySessionsScript = `
local readySessionsKey = KEYS[1]
local sessionPrefix = KEYS[2]
local sessionsSortedKey = KEYS[3]
local maxSessions = tonumber(ARGV[1])
local maxEventsPerSession = tonumber(ARGV[2])
local startOffset = tonumber(ARGV[3]) or 0
@@ -117,7 +118,7 @@ for i, sessionId in ipairs(sessionIds) do
local eventCount = redis.call('LLEN', sessionKey)
if eventCount == 0 then
-- Session is empty, remove from ready set
-- Session is empty, remove from both sets
table.insert(sessionsToRemove, sessionId)
else
-- Fetch limited number of events to avoid huge payloads
@@ -133,9 +134,14 @@ for i, sessionId in ipairs(sessionIds) do
end
end
-- Clean up empty sessions from ready set
-- Clean up empty sessions from both ready set and sorted set
if #sessionsToRemove > 0 then
redis.call('ZREM', readySessionsKey, unpack(sessionsToRemove))
redis.call('ZREM', sessionsSortedKey, unpack(sessionsToRemove))
-- Also delete the empty session keys
for i, sessionId in ipairs(sessionsToRemove) do
redis.call('DEL', sessionPrefix .. sessionId)
end
end
return cjson.encode(result)
@@ -147,7 +153,8 @@ return cjson.encode(result)
*
* KEYS[1] = session key
* KEYS[2] = ready sessions key
* KEYS[3] = buffer counter key
* KEYS[3] = sessions sorted key
* KEYS[4] = buffer counter key
* ARGV[1] = sessionId
* ARGV[2] = snapshotCount (number of events that were present in our snapshot)
* ARGV[3] = pendingCount (number of pending events)
@@ -157,7 +164,8 @@ return cjson.encode(result)
private readonly updateSessionScript = `
local sessionKey = KEYS[1]
local readySessionsKey = KEYS[2]
local bufferCounterKey = KEYS[3]
local sessionsSortedKey = KEYS[3]
local bufferCounterKey = KEYS[4]
local sessionId = ARGV[1]
local snapshotCount = tonumber(ARGV[2])
local pendingCount = tonumber(ARGV[3])
@@ -174,9 +182,17 @@ end
local newLength = redis.call("LLEN", sessionKey)
-- Update ready sessions set based on new length
if newLength >= minEventsInSession then
redis.call("ZADD", readySessionsKey, "XX", redis.call("TIME")[1], sessionId)
if newLength == 0 then
-- Session is now empty, remove from both sets and delete key
redis.call("ZREM", readySessionsKey, sessionId)
redis.call("ZREM", sessionsSortedKey, sessionId)
redis.call("DEL", sessionKey)
elseif newLength >= minEventsInSession then
-- Session has enough events, keep/add it in ready_sessions
redis.call("ZADD", readySessionsKey, redis.call("TIME")[1], sessionId)
else
-- Session has events but < minEvents, remove from ready_sessions
-- It will be re-added when a new event arrives (via addEventScript)
redis.call("ZREM", readySessionsKey, sessionId)
end
@@ -192,12 +208,14 @@ return newLength
/**
* Optimized batch update script with counter and ready sessions management.
* KEYS[1] = ready sessions key
* KEYS[2] = buffer counter key
* ARGV format: [sessionKey1, sessionId1, snapshotCount1, pendingCount1, pending1...., sessionKey2, ...]
* KEYS[2] = sessions sorted key
* KEYS[3] = buffer counter key
* ARGV format: [minEventsInSession, sessionKey1, sessionId1, snapshotCount1, pendingCount1, pending1...., sessionKey2, ...]
*/
private readonly batchUpdateSessionsScript = `
local readySessionsKey = KEYS[1]
local bufferCounterKey = KEYS[2]
local sessionsSortedKey = KEYS[2]
local bufferCounterKey = KEYS[3]
local minEventsInSession = tonumber(ARGV[1])
local totalCounterChange = 0
@@ -222,9 +240,17 @@ while i <= #ARGV do
local newLength = redis.call("LLEN", sessionKey)
-- Update ready sessions set based on new length
if newLength >= minEventsInSession then
redis.call("ZADD", readySessionsKey, "XX", redis.call("TIME")[1], sessionId)
if newLength == 0 then
-- Session is now empty, remove from both sets and delete key
redis.call("ZREM", readySessionsKey, sessionId)
redis.call("ZREM", sessionsSortedKey, sessionId)
redis.call("DEL", sessionKey)
elseif newLength >= minEventsInSession then
-- Session has enough events, keep/add it in ready_sessions
redis.call("ZADD", readySessionsKey, redis.call("TIME")[1], sessionId)
else
-- Session has events but < minEvents, remove from ready_sessions
-- It will be re-added when a new event arrives (via addEventScript)
redis.call("ZREM", readySessionsKey, sessionId)
end
@@ -398,9 +424,10 @@ return "OK"
) {
const sessionsSorted = await getRedisCache().eval(
this.processReadySessionsScript,
2, // number of KEYS
3, // number of KEYS
this.readySessionsKey,
this.sessionKeyPrefix,
this.sessionSortedKey,
sessionsPerPage.toString(),
maxEventsPerSession.toString(),
startOffset.toString(),
@@ -646,29 +673,31 @@ return "OK"
const flush: IClickhouseEvent[] = [];
const pending: IClickhouseEvent[] = [];
let hasSessionEnd = false;
// Check if session has ended - if so, flush everything
const hasSessionEnd = events.some((e) => e.name === 'session_end');
if (hasSessionEnd) {
flush.push(...events);
return { flush, pending: [] };
}
const findNextScreenView = (events: IClickhouseEvent[]) => {
return events.find((e) => e.name === 'screen_view');
};
for (let i = 0; i < events.length; i++) {
const event = events[i]!;
if (event.name === 'session_end') {
hasSessionEnd = true;
// For screen_view events, look for next event
const next = findNextScreenView(events.slice(i + 1));
if (next) {
event.duration =
new Date(next.created_at).getTime() -
new Date(event.created_at).getTime();
flush.push(event);
} else {
// For screen_view events, look for next event
const next = events[i + 1];
if (next) {
if (next.name === 'screen_view') {
event.duration =
new Date(next.created_at).getTime() -
new Date(event.created_at).getTime();
}
flush.push(event);
} else if (hasSessionEnd) {
flush.push(event);
} else {
pending.push(event);
}
// Last screen_view with no next event - keep pending
pending.push(event);
}
}
@@ -696,23 +725,58 @@ return "OK"
const cutoffTime = Date.now() - 1000 * 60 * 60 * 24 * this.daysToKeep;
try {
const sessionIds = await redis.zrange(this.sessionSortedKey, 0, -1);
const sessionCount = await redis.zcard(this.sessionSortedKey);
const batchSize = 1000;
let offset = 0;
let totalCleaned = 0;
for (const sessionId of sessionIds) {
const score = await redis.zscore(this.sessionSortedKey, sessionId);
this.logger.info('Starting cleanup of stale sessions', {
cutoffTime: new Date(cutoffTime),
totalSessions: sessionCount,
});
if (score) {
const scoreInt = Number.parseInt(score, 10);
if (scoreInt < cutoffTime) {
this.logger.warn('Stale session found', {
sessionId,
score,
createdAt: new Date(Number.parseInt(score, 10)),
eventsCount: await redis.llen(this.getSessionKey(sessionId)),
});
while (offset < sessionCount) {
// Get batch of session IDs with scores
const sessionIdsWithScores = await redis.zrange(
this.sessionSortedKey,
offset,
offset + batchSize - 1,
'WITHSCORES',
);
if (sessionIdsWithScores.length === 0) break;
const pipeline = redis.pipeline();
let staleSessions = 0;
// Process pairs of [sessionId, score]
for (let i = 0; i < sessionIdsWithScores.length; i += 2) {
const sessionId = sessionIdsWithScores[i];
const score = Number.parseInt(sessionIdsWithScores[i + 1] || '0', 10);
if (sessionId && score < cutoffTime) {
staleSessions++;
// Remove from both sorted sets and delete the session key
pipeline.zrem(this.sessionSortedKey, sessionId);
pipeline.zrem(this.readySessionsKey, sessionId);
pipeline.del(this.getSessionKey(sessionId));
}
}
if (staleSessions > 0) {
await pipeline.exec();
totalCleaned += staleSessions;
this.logger.info('Cleaned batch of stale sessions', {
batch: Math.floor(offset / batchSize) + 1,
cleanedInBatch: staleSessions,
totalCleaned,
});
}
offset += batchSize;
}
this.logger.info('Cleanup completed', { totalCleaned });
} catch (error) {
this.logger.error('Failed to cleanup stale sessions', { error });
}
@@ -798,8 +862,9 @@ return "OK"
await redis.eval(
this.batchUpdateSessionsScript,
2, // KEYS: ready sessions, buffer counter
3, // KEYS: ready sessions, sessions sorted, buffer counter
this.readySessionsKey,
this.sessionSortedKey,
this.bufferCounterKey,
...batchArgs,
);
@@ -809,40 +874,52 @@ return "OK"
public async getBufferSizeHeavy() {
// Fallback method for when counter is not available
const redis = getRedisCache();
const pipeline = redis.pipeline();
// Queue up commands in the pipeline
pipeline.llen(this.regularQueueKey);
pipeline.zcard(this.sessionSortedKey);
// Get regular queue count
const regularQueueCount = await redis.llen(this.regularQueueKey);
// Execute pipeline to get initial counts
const [regularQueueCount, sessionCount] = (await pipeline.exec()) as [
any,
any,
];
// Get total number of sessions
const sessionCount = await redis.zcard(this.sessionSortedKey);
if (sessionCount[1] === 0) {
return regularQueueCount[1];
if (sessionCount === 0) {
return regularQueueCount;
}
// Get all session IDs and queue up LLEN commands for each session
const sessionIds = await redis.zrange(this.sessionSortedKey, 0, -1);
const sessionPipeline = redis.pipeline();
// Process sessions in batches to avoid memory spikes
const batchSize = 1000;
let totalSessionEvents = 0;
let offset = 0;
for (const sessionId of sessionIds) {
sessionPipeline.llen(this.getSessionKey(sessionId));
while (offset < sessionCount) {
// Get batch of session IDs
const sessionIds = await redis.zrange(
this.sessionSortedKey,
offset,
offset + batchSize - 1,
);
if (sessionIds.length === 0) break;
// Queue up LLEN commands for this batch
const sessionPipeline = redis.pipeline();
for (const sessionId of sessionIds) {
sessionPipeline.llen(this.getSessionKey(sessionId));
}
// Execute pipeline for this batch
const sessionCounts = (await sessionPipeline.exec()) as [any, any][];
// Sum up counts from this batch
for (const [err, count] of sessionCounts) {
if (!err) {
totalSessionEvents += count;
}
}
offset += batchSize;
}
// Execute all LLEN commands in a single pipeline
const sessionCounts = (await sessionPipeline.exec()) as [any, any][];
// Sum up all counts
const totalSessionEvents = sessionCounts.reduce((sum, [err, count]) => {
if (err) return sum;
return sum + count;
}, 0);
return regularQueueCount[1] + totalSessionEvents;
return regularQueueCount + totalSessionEvents;
}
public async getBufferSize() {

View File

@@ -10,7 +10,7 @@
"@openpanel/logger": "workspace:*",
"@openpanel/redis": "workspace:*",
"bullmq": "^5.8.7",
"groupmq": "1.0.0-next.14"
"groupmq": "1.0.0-next.15"
},
"devDependencies": {
"@openpanel/sdk": "workspace:*",

View File

@@ -113,7 +113,8 @@ export const eventsGroupQueue = new GroupQueue<
logger: queueLogger,
namespace: 'group_events',
redis: getRedisGroupQueue(),
orderingDelayMs: 2000,
orderingMethod: 'in-memory',
orderingWindowMs: 50,
keepCompleted: 10,
keepFailed: 10_000,
});