zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: optimize event buffer (#278)

Browse Source 
* fix: how we fetch profiles in the buffer

* perf: optimize event buffer

* remove unused file

* fix

* wip

* wip: try groupmq 2

* try simplified event buffer with duration calculation on the fly instead
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-16 13:29:40 +01:00
committed by GitHub
parent 4736f8509d
commit 4483e464d1
46 changed files with 887 additions and 1841 deletions
Download Patch File Download Diff File Expand all files Collapse all files

366
packages/db/src/buffers/event-buffer.test.ts
View File

@@ -2,42 +2,8 @@ import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { ch } from '../clickhouse/client';
// Mock transformEvent to avoid circular dependency with buffers -> services -> buffers
vi.mock('../services/event.service', () => ({
transformEvent: (event: any) => ({
id: event.id ?? 'id',
name: event.name,
deviceId: event.device_id,
profileId: event.profile_id,
projectId: event.project_id,
sessionId: event.session_id,
properties: event.properties ?? {},
createdAt: new Date(event.created_at ?? Date.now()),
country: event.country,
city: event.city,
region: event.region,
longitude: event.longitude,
latitude: event.latitude,
os: event.os,
osVersion: event.os_version,
browser: event.browser,
browserVersion: event.browser_version,
device: event.device,
brand: event.brand,
model: event.model,
duration: event.duration ?? 0,
path: event.path ?? '',
origin: event.origin ?? '',
referrer: event.referrer,
referrerName: event.referrer_name,
referrerType: event.referrer_type,
meta: event.meta,
importedAt: undefined,
sdkName: event.sdk_name,
sdkVersion: event.sdk_version,
profile: event.profile,
}),
}));
// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
vi.mock('../services/event.service', () => ({}));
import { EventBuffer } from './event-buffer';
@@ -68,18 +34,16 @@ describe('EventBuffer', () => {
created_at: new Date().toISOString(),
} as any;
// Get initial count
const initialCount = await eventBuffer.getBufferSize();
// Add event
await eventBuffer.add(event);
eventBuffer.add(event);
await eventBuffer.flush();
// Buffer counter should increase by 1
const newCount = await eventBuffer.getBufferSize();
expect(newCount).toBe(initialCount + 1);
});
it('adds multiple screen_views - moves previous to buffer with duration', async () => {
it('adds screen_view directly to buffer queue', async () => {
const t0 = Date.now();
const sessionId = 'session_1';
@@ -99,60 +63,23 @@ describe('EventBuffer', () => {
created_at: new Date(t0 + 1000).toISOString(),
} as any;
const view3 = {
project_id: 'p1',
profile_id: 'u1',
session_id: sessionId,
name: 'screen_view',
created_at: new Date(t0 + 3000).toISOString(),
} as any;
// Add first screen_view
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(view1);
// Should be stored as "last" but NOT in queue yet
eventBuffer.add(view1);
await eventBuffer.flush();
// screen_view goes directly to buffer
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1); // No change in buffer
expect(count2).toBe(count1 + 1);
// Last screen_view should be retrievable
const last1 = await eventBuffer.getLastScreenView({
projectId: 'p1',
sessionId: sessionId,
});
expect(last1).not.toBeNull();
expect(last1!.createdAt.toISOString()).toBe(view1.created_at);
eventBuffer.add(view2);
await eventBuffer.flush();
// Add second screen_view
await eventBuffer.add(view2);
// Now view1 should be in buffer
const count3 = await eventBuffer.getBufferSize();
expect(count3).toBe(count1 + 1);
// view2 should now be the "last"
const last2 = await eventBuffer.getLastScreenView({
projectId: 'p1',
sessionId: sessionId,
});
expect(last2!.createdAt.toISOString()).toBe(view2.created_at);
// Add third screen_view
await eventBuffer.add(view3);
// Now view2 should also be in buffer
const count4 = await eventBuffer.getBufferSize();
expect(count4).toBe(count1 + 2);
// view3 should now be the "last"
const last3 = await eventBuffer.getLastScreenView({
projectId: 'p1',
sessionId: sessionId,
});
expect(last3!.createdAt.toISOString()).toBe(view3.created_at);
expect(count3).toBe(count1 + 2);
});
it('adds session_end - moves last screen_view and session_end to buffer', async () => {
it('adds session_end directly to buffer queue', async () => {
const t0 = Date.now();
const sessionId = 'session_2';
@@ -172,148 +99,44 @@ describe('EventBuffer', () => {
created_at: new Date(t0 + 5000).toISOString(),
} as any;
// Add screen_view
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(view);
// Should be stored as "last", not in buffer yet
eventBuffer.add(view);
eventBuffer.add(sessionEnd);
await eventBuffer.flush();
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1);
// Add session_end
await eventBuffer.add(sessionEnd);
// Both should now be in buffer (+2)
const count3 = await eventBuffer.getBufferSize();
expect(count3).toBe(count1 + 2);
// Last screen_view should be cleared
const last = await eventBuffer.getLastScreenView({
projectId: 'p2',
sessionId: sessionId,
});
expect(last).toBeNull();
});
it('session_end with no previous screen_view - only adds session_end to buffer', async () => {
const sessionId = 'session_3';
const sessionEnd = {
project_id: 'p3',
profile_id: 'u3',
session_id: sessionId,
name: 'session_end',
created_at: new Date().toISOString(),
} as any;
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(sessionEnd);
// Only session_end should be in buffer (+1)
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1 + 1);
});
it('gets last screen_view by profileId', async () => {
const view = {
project_id: 'p4',
profile_id: 'u4',
session_id: 'session_4',
name: 'screen_view',
path: '/home',
created_at: new Date().toISOString(),
} as any;
await eventBuffer.add(view);
// Query by profileId
const result = await eventBuffer.getLastScreenView({
projectId: 'p4',
profileId: 'u4',
});
expect(result).not.toBeNull();
expect(result!.name).toBe('screen_view');
expect(result!.path).toBe('/home');
});
it('gets last screen_view by sessionId', async () => {
const sessionId = 'session_5';
const view = {
project_id: 'p5',
profile_id: 'u5',
session_id: sessionId,
name: 'screen_view',
path: '/about',
created_at: new Date().toISOString(),
} as any;
await eventBuffer.add(view);
// Query by sessionId
const result = await eventBuffer.getLastScreenView({
projectId: 'p5',
sessionId: sessionId,
});
expect(result).not.toBeNull();
expect(result!.name).toBe('screen_view');
expect(result!.path).toBe('/about');
});
it('returns null for non-existent last screen_view', async () => {
const result = await eventBuffer.getLastScreenView({
projectId: 'p_nonexistent',
profileId: 'u_nonexistent',
});
expect(result).toBeNull();
expect(count2).toBe(count1 + 2);
});
it('gets buffer count correctly', async () => {
// Initially 0
expect(await eventBuffer.getBufferSize()).toBe(0);
// Add regular event
await eventBuffer.add({
eventBuffer.add({
project_id: 'p6',
name: 'event1',
created_at: new Date().toISOString(),
} as any);
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(1);
// Add another regular event
await eventBuffer.add({
eventBuffer.add({
project_id: 'p6',
name: 'event2',
created_at: new Date().toISOString(),
} as any);
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(2);
// Add screen_view (not counted until flushed)
await eventBuffer.add({
// screen_view also goes directly to buffer
eventBuffer.add({
project_id: 'p6',
profile_id: 'u6',
session_id: 'session_6',
name: 'screen_view',
created_at: new Date().toISOString(),
} as any);
// Still 2 (screen_view is pending)
expect(await eventBuffer.getBufferSize()).toBe(2);
// Add another screen_view (first one gets flushed)
await eventBuffer.add({
project_id: 'p6',
profile_id: 'u6',
session_id: 'session_6',
name: 'screen_view',
created_at: new Date(Date.now() + 1000).toISOString(),
} as any);
// Now 3 (2 regular + 1 flushed screen_view)
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(3);
});
@@ -330,8 +153,9 @@ describe('EventBuffer', () => {
created_at: new Date(Date.now() + 1000).toISOString(),
} as any;
await eventBuffer.add(event1);
await eventBuffer.add(event2);
eventBuffer.add(event1);
eventBuffer.add(event2);
await eventBuffer.flush();
expect(await eventBuffer.getBufferSize()).toBe(2);
@@ -341,14 +165,12 @@ describe('EventBuffer', () => {
await eventBuffer.processBuffer();
// Should insert both events
expect(insertSpy).toHaveBeenCalled();
const callArgs = insertSpy.mock.calls[0]![0];
expect(callArgs.format).toBe('JSONEachRow');
expect(callArgs.table).toBe('events');
expect(Array.isArray(callArgs.values)).toBe(true);
// Buffer should be empty after processing
expect(await eventBuffer.getBufferSize()).toBe(0);
insertSpy.mockRestore();
@@ -359,14 +181,14 @@ describe('EventBuffer', () => {
process.env.EVENT_BUFFER_CHUNK_SIZE = '2';
const eb = new EventBuffer();
// Add 4 events
for (let i = 0; i < 4; i++) {
await eb.add({
eb.add({
project_id: 'p8',
name: `event${i}`,
created_at: new Date(Date.now() + i).toISOString(),
} as any);
}
await eb.flush();
const insertSpy = vi
.spyOn(ch, 'insert')
@@ -374,14 +196,12 @@ describe('EventBuffer', () => {
await eb.processBuffer();
// With chunk size 2 and 4 events, should be called twice
expect(insertSpy).toHaveBeenCalledTimes(2);
const call1Values = insertSpy.mock.calls[0]![0].values as any[];
const call2Values = insertSpy.mock.calls[1]![0].values as any[];
expect(call1Values.length).toBe(2);
expect(call2Values.length).toBe(2);
// Restore
if (prev === undefined) delete process.env.EVENT_BUFFER_CHUNK_SIZE;
else process.env.EVENT_BUFFER_CHUNK_SIZE = prev;
@@ -396,129 +216,61 @@ describe('EventBuffer', () => {
created_at: new Date().toISOString(),
} as any;
await eventBuffer.add(event);
eventBuffer.add(event);
await eventBuffer.flush();
const count = await eventBuffer.getActiveVisitorCount('p9');
expect(count).toBeGreaterThanOrEqual(1);
});
it('handles multiple sessions independently', async () => {
it('handles multiple sessions independently — all events go to buffer', async () => {
const t0 = Date.now();
const count1 = await eventBuffer.getBufferSize();
// Session 1
const view1a = {
eventBuffer.add({
project_id: 'p10',
profile_id: 'u10',
session_id: 'session_10a',
name: 'screen_view',
created_at: new Date(t0).toISOString(),
} as any;
const view1b = {
project_id: 'p10',
profile_id: 'u10',
session_id: 'session_10a',
name: 'screen_view',
created_at: new Date(t0 + 1000).toISOString(),
} as any;
// Session 2
const view2a = {
} as any);
eventBuffer.add({
project_id: 'p10',
profile_id: 'u11',
session_id: 'session_10b',
name: 'screen_view',
created_at: new Date(t0).toISOString(),
} as any;
const view2b = {
} as any);
eventBuffer.add({
project_id: 'p10',
profile_id: 'u10',
session_id: 'session_10a',
name: 'screen_view',
created_at: new Date(t0 + 1000).toISOString(),
} as any);
eventBuffer.add({
project_id: 'p10',
profile_id: 'u11',
session_id: 'session_10b',
name: 'screen_view',
created_at: new Date(t0 + 2000).toISOString(),
} as any;
} as any);
await eventBuffer.flush();
await eventBuffer.add(view1a);
await eventBuffer.add(view2a);
await eventBuffer.add(view1b); // Flushes view1a
await eventBuffer.add(view2b); // Flushes view2a
// Should have 2 events in buffer (one from each session)
expect(await eventBuffer.getBufferSize()).toBe(2);
// Each session should have its own "last" screen_view
const last1 = await eventBuffer.getLastScreenView({
projectId: 'p10',
sessionId: 'session_10a',
});
expect(last1!.createdAt.toISOString()).toBe(view1b.created_at);
const last2 = await eventBuffer.getLastScreenView({
projectId: 'p10',
sessionId: 'session_10b',
});
expect(last2!.createdAt.toISOString()).toBe(view2b.created_at);
// All 4 events are in buffer directly
expect(await eventBuffer.getBufferSize()).toBe(count1 + 4);
});
it('screen_view without session_id goes directly to buffer', async () => {
const view = {
it('bulk adds events to buffer', async () => {
const events = Array.from({ length: 5 }, (_, i) => ({
project_id: 'p11',
profile_id: 'u11',
name: 'screen_view',
created_at: new Date().toISOString(),
} as any;
name: `event${i}`,
created_at: new Date(Date.now() + i).toISOString(),
})) as any[];
const count1 = await eventBuffer.getBufferSize();
await eventBuffer.add(view);
eventBuffer.bulkAdd(events);
await eventBuffer.flush();
// Should go directly to buffer (no session_id)
const count2 = await eventBuffer.getBufferSize();
expect(count2).toBe(count1 + 1);
});
it('updates last screen_view when new one arrives from same profile but different session', async () => {
const t0 = Date.now();
const view1 = {
project_id: 'p12',
profile_id: 'u12',
session_id: 'session_12a',
name: 'screen_view',
path: '/page1',
created_at: new Date(t0).toISOString(),
} as any;
const view2 = {
project_id: 'p12',
profile_id: 'u12',
session_id: 'session_12b', // Different session!
name: 'screen_view',
path: '/page2',
created_at: new Date(t0 + 1000).toISOString(),
} as any;
await eventBuffer.add(view1);
await eventBuffer.add(view2);
// Both sessions should have their own "last"
const lastSession1 = await eventBuffer.getLastScreenView({
projectId: 'p12',
sessionId: 'session_12a',
});
expect(lastSession1!.path).toBe('/page1');
const lastSession2 = await eventBuffer.getLastScreenView({
projectId: 'p12',
sessionId: 'session_12b',
});
expect(lastSession2!.path).toBe('/page2');
// Profile should have the latest one
const lastProfile = await eventBuffer.getLastScreenView({
projectId: 'p12',
profileId: 'u12',
});
expect(lastProfile!.path).toBe('/page2');
expect(await eventBuffer.getBufferSize()).toBe(5);
});
});

442
packages/db/src/buffers/event-buffer.ts
View File

@@ -2,33 +2,13 @@ import { getSafeJson } from '@openpanel/json';
import {
type Redis,
getRedisCache,
getRedisPub,
publishEvent,
} from '@openpanel/redis';
import { ch } from '../clickhouse/client';
import {
type IClickhouseEvent,
type IServiceEvent,
transformEvent,
} from '../services/event.service';
import { type IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
/**
* Simplified Event Buffer
*
* Rules:
* 1. All events go into a single list buffer (event_buffer:queue)
* 2. screen_view events are handled specially:
* - Store current screen_view as "last" for the session
* - When a new screen_view arrives, flush the previous one with calculated duration
* 3. session_end events:
* - Retrieve the last screen_view (don't modify it)
* - Push both screen_view and session_end to buffer
* 4. Flush: Simply process all events from the list buffer
*/
export class EventBuffer extends BaseBuffer {
// Configurable limits
private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.EVENT_BUFFER_BATCH_SIZE, 10)
: 4000;
@@ -36,124 +16,26 @@ export class EventBuffer extends BaseBuffer {
? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)
: 1000;
private microBatchIntervalMs = process.env.EVENT_BUFFER_MICRO_BATCH_MS
? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_MS, 10)
: 10;
private microBatchMaxSize = process.env.EVENT_BUFFER_MICRO_BATCH_SIZE
? Number.parseInt(process.env.EVENT_BUFFER_MICRO_BATCH_SIZE, 10)
: 100;
private pendingEvents: IClickhouseEvent[] = [];
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private isFlushing = false;
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
private activeVisitorsExpiration = 60 * 5; // 5 minutes
// LIST - Stores all events ready to be flushed
/** How often (ms) we refresh the heartbeat key + zadd per visitor. */
private heartbeatRefreshMs = 60_000; // 1 minute
private lastHeartbeat = new Map<string, number>();
private queueKey = 'event_buffer:queue';
// STRING - Tracks total buffer size incrementally
protected bufferCounterKey = 'event_buffer:total_count';
// Script SHAs for loaded Lua scripts
private scriptShas: {
addScreenView?: string;
addSessionEnd?: string;
} = {};
// Hash key for storing last screen_view per session
private getLastScreenViewKeyBySession(sessionId: string) {
return `event_buffer:last_screen_view:session:${sessionId}`;
}
// Hash key for storing last screen_view per profile
private getLastScreenViewKeyByProfile(projectId: string, profileId: string) {
return `event_buffer:last_screen_view:profile:${projectId}:${profileId}`;
}
/**
* Lua script for handling screen_view addition - RACE-CONDITION SAFE without GroupMQ
*
* Strategy: Use Redis GETDEL (atomic get-and-delete) to ensure only ONE thread
* can process the "last" screen_view at a time.
*
* KEYS[1] = last screen_view key (by session) - stores both event and timestamp as JSON
* KEYS[2] = last screen_view key (by profile, may be empty)
* KEYS[3] = queue key
* KEYS[4] = buffer counter key
* ARGV[1] = new event with timestamp as JSON: {"event": {...}, "ts": 123456}
* ARGV[2] = TTL for last screen_view (1 hour)
*/
private readonly addScreenViewScript = `
local sessionKey = KEYS[1]
local profileKey = KEYS[2]
local queueKey = KEYS[3]
local counterKey = KEYS[4]
local newEventData = ARGV[1]
local ttl = tonumber(ARGV[2])
-- GETDEL is atomic: get previous and delete in one operation
-- This ensures only ONE thread gets the previous event
local previousEventData = redis.call("GETDEL", sessionKey)
-- Store new screen_view as last for session
redis.call("SET", sessionKey, newEventData, "EX", ttl)
-- Store new screen_view as last for profile (if key provided)
if profileKey and profileKey ~= "" then
redis.call("SET", profileKey, newEventData, "EX", ttl)
end
-- If there was a previous screen_view, add it to queue with calculated duration
if previousEventData then
local prev = cjson.decode(previousEventData)
local curr = cjson.decode(newEventData)
-- Calculate duration (ensure non-negative to handle clock skew)
if prev.ts and curr.ts then
prev.event.duration = math.max(0, curr.ts - prev.ts)
end
redis.call("RPUSH", queueKey, cjson.encode(prev.event))
redis.call("INCR", counterKey)
return 1
end
return 0
`;
/**
* Lua script for handling session_end - RACE-CONDITION SAFE
*
* Uses GETDEL to atomically retrieve and delete the last screen_view
*
* KEYS[1] = last screen_view key (by session)
* KEYS[2] = last screen_view key (by profile, may be empty)
* KEYS[3] = queue key
* KEYS[4] = buffer counter key
* ARGV[1] = session_end event JSON
*/
private readonly addSessionEndScript = `
local sessionKey = KEYS[1]
local profileKey = KEYS[2]
local queueKey = KEYS[3]
local counterKey = KEYS[4]
local sessionEndJson = ARGV[1]
-- GETDEL is atomic: only ONE thread gets the last screen_view
local previousEventData = redis.call("GETDEL", sessionKey)
local added = 0
-- If there was a previous screen_view, add it to queue
if previousEventData then
local prev = cjson.decode(previousEventData)
redis.call("RPUSH", queueKey, cjson.encode(prev.event))
redis.call("INCR", counterKey)
added = added + 1
end
-- Add session_end to queue
redis.call("RPUSH", queueKey, sessionEndJson)
redis.call("INCR", counterKey)
added = added + 1
-- Delete profile key
if profileKey and profileKey ~= "" then
redis.call("DEL", profileKey)
end
return added
`;
constructor() {
super({
name: 'event',
@@ -161,170 +43,97 @@ return added
await this.processBuffer();
},
});
// Load Lua scripts into Redis on startup
this.loadScripts();
}
/**
* Load Lua scripts into Redis and cache their SHAs.
* This avoids sending the entire script on every call.
*/
private async loadScripts() {
try {
const redis = getRedisCache();
const [screenViewSha, sessionEndSha] = await Promise.all([
redis.script('LOAD', this.addScreenViewScript),
redis.script('LOAD', this.addSessionEndScript),
]);
this.scriptShas.addScreenView = screenViewSha as string;
this.scriptShas.addSessionEnd = sessionEndSha as string;
this.logger.info('Loaded Lua scripts into Redis', {
addScreenView: this.scriptShas.addScreenView,
addSessionEnd: this.scriptShas.addSessionEnd,
});
} catch (error) {
this.logger.error('Failed to load Lua scripts', { error });
}
}
bulkAdd(events: IClickhouseEvent[]) {
const redis = getRedisCache();
const multi = redis.multi();
for (const event of events) {
this.add(event, multi);
this.add(event);
}
return multi.exec();
}
/**
* Add an event into Redis buffer.
*
* Logic:
* - screen_view: Store as "last" for session, flush previous if exists
* - session_end: Flush last screen_view + session_end
* - Other events: Add directly to queue
*/
async add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
add(event: IClickhouseEvent) {
this.pendingEvents.push(event);
if (this.pendingEvents.length >= this.microBatchMaxSize) {
this.flushLocalBuffer();
return;
}
if (!this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
public async flush() {
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
await this.flushLocalBuffer();
}
private async flushLocalBuffer() {
if (this.isFlushing || this.pendingEvents.length === 0) {
return;
}
this.isFlushing = true;
const eventsToFlush = this.pendingEvents;
this.pendingEvents = [];
try {
const redis = getRedisCache();
const eventJson = JSON.stringify(event);
const multi = _multi || redis.multi();
const multi = redis.multi();
if (event.session_id && event.name === 'screen_view') {
// Handle screen_view
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
const profileKey = event.profile_id
? this.getLastScreenViewKeyByProfile(
event.project_id,
event.profile_id,
)
: '';
const timestamp = new Date(event.created_at || Date.now()).getTime();
// Combine event and timestamp into single JSON for atomic operations
const eventWithTimestamp = JSON.stringify({
event: event,
ts: timestamp,
});
this.evalScript(
multi,
'addScreenView',
this.addScreenViewScript,
4,
sessionKey,
profileKey,
this.queueKey,
this.bufferCounterKey,
eventWithTimestamp,
'3600', // 1 hour TTL
);
} else if (event.session_id && event.name === 'session_end') {
// Handle session_end
const sessionKey = this.getLastScreenViewKeyBySession(event.session_id);
const profileKey = event.profile_id
? this.getLastScreenViewKeyByProfile(
event.project_id,
event.profile_id,
)
: '';
this.evalScript(
multi,
'addSessionEnd',
this.addSessionEndScript,
4,
sessionKey,
profileKey,
this.queueKey,
this.bufferCounterKey,
eventJson,
);
} else {
// All other events go directly to queue
multi.rpush(this.queueKey, eventJson).incr(this.bufferCounterKey);
for (const event of eventsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(event));
if (event.profile_id) {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id,
);
}
}
multi.incrby(this.bufferCounterKey, eventsToFlush.length);
if (event.profile_id) {
this.incrementActiveVisitorCount(
multi,
event.project_id,
event.profile_id,
);
}
await multi.exec();
if (!_multi) {
await multi.exec();
}
await publishEvent('events', 'received', transformEvent(event));
this.flushRetryCount = 0;
this.pruneHeartbeatMap();
} catch (error) {
this.logger.error('Failed to add event to Redis buffer', { error });
// Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
this.flushRetryCount += 1;
this.logger.warn(
'Failed to flush local buffer to Redis; events re-queued',
{
error,
eventCount: eventsToFlush.length,
flushRetryCount: this.flushRetryCount,
},
);
} finally {
this.isFlushing = false;
// Events may have accumulated while we were flushing; schedule another flush if needed
if (this.pendingEvents.length > 0 && !this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
}
/**
* Execute a Lua script using EVALSHA (cached) or fallback to EVAL.
* This avoids sending the entire script on every call.
*/
private evalScript(
multi: ReturnType<Redis['multi']>,
scriptName: keyof typeof this.scriptShas,
scriptContent: string,
numKeys: number,
...args: (string | number)[]
) {
const sha = this.scriptShas[scriptName];
if (sha) {
// Use EVALSHA with cached SHA
multi.evalsha(sha, numKeys, ...args);
} else {
// Fallback to EVAL and try to reload script
multi.eval(scriptContent, numKeys, ...args);
this.logger.warn(`Script ${scriptName} not loaded, using EVAL fallback`);
// Attempt to reload scripts in background
this.loadScripts();
}
}
/**
* Process the Redis buffer - simplified version.
*
* Simply:
* 1. Fetch events from the queue (up to batchSize)
* 2. Parse and sort them
* 3. Insert into ClickHouse in chunks
* 4. Publish saved events
* 5. Clean up processed events from queue
*/
async processBuffer() {
const redis = getRedisCache();
try {
// Fetch events from queue
const queueEvents = await redis.lrange(
this.queueKey,
0,
@@ -336,7 +145,6 @@ return added
return;
}
// Parse events
const eventsToClickhouse: IClickhouseEvent[] = [];
for (const eventStr of queueEvents) {
const event = getSafeJson<IClickhouseEvent>(eventStr);
@@ -350,14 +158,12 @@ return added
return;
}
// Sort events by creation time
eventsToClickhouse.sort(
(a, b) =>
new Date(a.created_at || 0).getTime() -
new Date(b.created_at || 0).getTime(),
);
// Insert events into ClickHouse in chunks
this.logger.info('Inserting events into ClickHouse', {
totalEvents: eventsToClickhouse.length,
chunks: Math.ceil(eventsToClickhouse.length / this.chunkSize),
@@ -371,14 +177,17 @@ return added
});
}
// Publish "saved" events
const pubMulti = getRedisPub().multi();
const countByProject = new Map<string, number>();
for (const event of eventsToClickhouse) {
await publishEvent('events', 'saved', transformEvent(event), pubMulti);
countByProject.set(
event.project_id,
(countByProject.get(event.project_id) ?? 0) + 1,
);
}
for (const [projectId, count] of countByProject) {
publishEvent('events', 'batch', { projectId, count });
}
await pubMulti.exec();
// Clean up processed events from queue
await redis
.multi()
.ltrim(this.queueKey, queueEvents.length, -1)
@@ -394,45 +203,6 @@ return added
}
}
/**
* Retrieve the latest screen_view event for a given session or profile
*/
public async getLastScreenView(
params:
| {
sessionId: string;
}
| {
projectId: string;
profileId: string;
},
): Promise<IServiceEvent | null> {
const redis = getRedisCache();
let lastScreenViewKey: string;
if ('sessionId' in params) {
lastScreenViewKey = this.getLastScreenViewKeyBySession(params.sessionId);
} else {
lastScreenViewKey = this.getLastScreenViewKeyByProfile(
params.projectId,
params.profileId,
);
}
const eventDataStr = await redis.get(lastScreenViewKey);
if (eventDataStr) {
const eventData = getSafeJson<{ event: IClickhouseEvent; ts: number }>(
eventDataStr,
);
if (eventData?.event) {
return transformEvent(eventData.event);
}
}
return null;
}
public async getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache();
@@ -440,16 +210,32 @@ return added
});
}
private async incrementActiveVisitorCount(
private pruneHeartbeatMap() {
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
for (const [key, ts] of this.lastHeartbeat) {
if (ts < cutoff) {
this.lastHeartbeat.delete(key);
}
}
}
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string,
) {
// Track active visitors and emit expiry events when inactive for TTL
const key = `${projectId}:${profileId}`;
const now = Date.now();
const last = this.lastHeartbeat.get(key) ?? 0;
if (now - last < this.heartbeatRefreshMs) {
return;
}
this.lastHeartbeat.set(key, now);
const zsetKey = `live:visitors:${projectId}`;
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
return multi
multi
.zadd(zsetKey, now, profileId)
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
}

40
packages/db/src/buffers/profile-buffer.ts
View File

@@ -1,11 +1,11 @@
import { deepMergeObjects } from '@openpanel/common';
import { getSafeJson } from '@openpanel/json';
import type { ILogger } from '@openpanel/logger';
import { type Redis, getRedisCache } from '@openpanel/redis';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import { omit } from 'ramda';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, ch, chQuery } from '../clickhouse/client';
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
import type { IClickhouseProfile } from '../services/profile.service';
import { BaseBuffer } from './base-buffer';
@@ -89,7 +89,7 @@ export class ProfileBuffer extends BaseBuffer {
'os_version',
'browser_version',
],
profile.properties,
profile.properties
);
}
@@ -97,16 +97,16 @@ export class ProfileBuffer extends BaseBuffer {
? deepMergeObjects(existingProfile, omit(['created_at'], profile))
: profile;
if (profile && existingProfile) {
if (
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile),
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
if (
profile &&
existingProfile &&
shallowEqual(
omit(['created_at'], existingProfile),
omit(['created_at'], mergedProfile)
)
) {
this.logger.debug('Profile not changed, skipping');
return;
}
this.logger.debug('Merged profile will be inserted', {
@@ -151,11 +151,11 @@ export class ProfileBuffer extends BaseBuffer {
private async fetchProfile(
profile: IClickhouseProfile,
logger: ILogger,
logger: ILogger
): Promise<IClickhouseProfile | null> {
const existingProfile = await this.fetchFromCache(
profile.id,
profile.project_id,
profile.project_id
);
if (existingProfile) {
logger.debug('Profile found in Redis');
@@ -167,7 +167,7 @@ export class ProfileBuffer extends BaseBuffer {
public async fetchFromCache(
profileId: string,
projectId: string,
projectId: string
): Promise<IClickhouseProfile | null> {
const cacheKey = this.getProfileCacheKey({
profileId,
@@ -182,7 +182,7 @@ export class ProfileBuffer extends BaseBuffer {
private async fetchFromClickhouse(
profile: IClickhouseProfile,
logger: ILogger,
logger: ILogger
): Promise<IClickhouseProfile | null> {
logger.debug('Fetching profile from Clickhouse');
const result = await chQuery<IClickhouseProfile>(
@@ -207,7 +207,7 @@ export class ProfileBuffer extends BaseBuffer {
}
GROUP BY id, project_id
ORDER BY created_at DESC
LIMIT 1`,
LIMIT 1`
);
logger.debug('Clickhouse fetch result', {
found: !!result[0],
@@ -221,7 +221,7 @@ export class ProfileBuffer extends BaseBuffer {
const profiles = await this.redis.lrange(
this.redisKey,
0,
this.batchSize - 1,
this.batchSize - 1
);
if (profiles.length === 0) {
@@ -231,7 +231,7 @@ export class ProfileBuffer extends BaseBuffer {
this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
const parsedProfiles = profiles.map((p) =>
getSafeJson<IClickhouseProfile>(p),
getSafeJson<IClickhouseProfile>(p)
);
for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {

7
packages/db/src/services/clients.service.ts
View File

@@ -1,4 +1,4 @@
import { cacheable, cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import type { Client, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
@@ -34,7 +34,4 @@ export async function getClientById(
});
}
export const getClientByIdCached = cacheableLru(getClientById, {
maxSize: 1000,
ttl: 60 * 5,
});
export const getClientByIdCached = cacheable(getClientById, 60 * 5);

16
packages/db/src/services/event.service.ts
View File

@@ -168,7 +168,6 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
device: event.device,
brand: event.brand,
model: event.model,
duration: event.duration,
path: event.path,
origin: event.origin,
referrer: event.referrer,
@@ -216,7 +215,7 @@ export interface IServiceEvent {
device?: string | undefined;
brand?: string | undefined;
model?: string | undefined;
duration: number;
duration?: number;
path: string;
origin: string;
referrer: string | undefined;
@@ -247,7 +246,7 @@ export interface IServiceEventMinimal {
browser?: string | undefined;
device?: string | undefined;
brand?: string | undefined;
duration: number;
duration?: number;
path: string;
origin: string;
referrer: string | undefined;
@@ -379,7 +378,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
device: payload.device ?? '',
brand: payload.brand ?? '',
model: payload.model ?? '',
duration: payload.duration,
duration: payload.duration ?? 0,
referrer: payload.referrer ?? '',
referrer_name: payload.referrerName ?? '',
referrer_type: payload.referrerType ?? '',
@@ -477,7 +476,7 @@ export async function getEventList(options: GetEventListOptions) {
sb.where.cursor = `created_at < ${sqlstring.escape(formatClickhouseDate(cursor))}`;
}
if (!cursor && !(startDate && endDate)) {
if (!(cursor || (startDate && endDate))) {
sb.where.cursorWindow = `created_at >= toDateTime64(${sqlstring.escape(formatClickhouseDate(new Date()))}, 3) - INTERVAL ${safeDateIntervalInDays} DAY`;
}
@@ -562,9 +561,6 @@ export async function getEventList(options: GetEventListOptions) {
if (select.model) {
sb.select.model = 'model';
}
if (select.duration) {
sb.select.duration = 'duration';
}
if (select.path) {
sb.select.path = 'path';
}
@@ -771,7 +767,6 @@ class EventService {
where,
select,
limit,
orderBy,
filters,
}: {
projectId: string;
@@ -811,7 +806,6 @@ class EventService {
select.event.deviceId && 'e.device_id as device_id',
select.event.name && 'e.name as name',
select.event.path && 'e.path as path',
select.event.duration && 'e.duration as duration',
select.event.country && 'e.country as country',
select.event.city && 'e.city as city',
select.event.os && 'e.os as os',
@@ -896,7 +890,6 @@ class EventService {
select.event.deviceId && 'e.device_id as device_id',
select.event.name && 'e.name as name',
select.event.path && 'e.path as path',
select.event.duration && 'e.duration as duration',
select.event.country && 'e.country as country',
select.event.city && 'e.city as city',
select.event.os && 'e.os as os',
@@ -1032,7 +1025,6 @@ class EventService {
id: true,
name: true,
createdAt: true,
duration: true,
country: true,
city: true,
os: true,

2
packages/db/src/services/notification.service.ts
View File

@@ -90,7 +90,7 @@ export const getNotificationRulesByProjectId = cacheable(
},
});
},
60 * 24
60 * 24,
);
function getIntegration(integrationId: string | null) {

35
packages/db/src/services/overview.service.ts
View File

@@ -416,6 +416,30 @@ export class OverviewService {
const where = this.getRawWhereClause('sessions', filters);
const fillConfig = this.getFillConfig(interval, startDate, endDate);
// CTE: per-event screen_view durations via window function
const rawScreenViewDurationsQuery = clix(this.client, timezone)
.select([
`${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
`dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
])
.from(TABLE_NAMES.events)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters));
// CTE: avg duration per date bucket
const avgDurationByDateQuery = clix(this.client, timezone)
.select([
'date',
'round(avgIf(duration, duration > 0), 2) / 1000 AS avg_session_duration',
])
.from('raw_screen_view_durations')
.groupBy(['date']);
// Session aggregation with bounce rates
const sessionAggQuery = clix(this.client, timezone)
.select([
@@ -473,6 +497,8 @@ export class OverviewService {
.where('date', '!=', rollupDate)
)
.with('overall_unique_visitors', overallUniqueVisitorsQuery)
.with('raw_screen_view_durations', rawScreenViewDurationsQuery)
.with('avg_duration_by_date', avgDurationByDateQuery)
.select<{
date: string;
bounce_rate: number;
@@ -489,8 +515,7 @@ export class OverviewService {
'dss.bounce_rate as bounce_rate',
'uniq(e.profile_id) AS unique_visitors',
'uniq(e.session_id) AS total_sessions',
'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
'coalesce(dur.avg_session_duration, 0) AS avg_session_duration',
'count(*) AS total_screen_views',
'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
'(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
@@ -502,6 +527,10 @@ export class OverviewService {
'daily_session_stats AS dss',
`${clix.toStartOf('e.created_at', interval as any)} = dss.date`
)
.leftJoin(
'avg_duration_by_date AS dur',
`${clix.toStartOf('e.created_at', interval as any)} = dur.date`
)
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.created_at', 'BETWEEN', [
@@ -509,7 +538,7 @@ export class OverviewService {
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters))
.groupBy(['date', 'dss.bounce_rate'])
.groupBy(['date', 'dss.bounce_rate', 'dur.avg_session_duration'])
.orderBy('date', 'ASC')
.fill(fillConfig.from, fillConfig.to, fillConfig.step)
.transform({

32
packages/db/src/services/pages.service.ts
View File

@@ -52,6 +52,24 @@ export class PagesService {
.where('created_at', '>=', clix.exp('now() - INTERVAL 30 DAY'))
.groupBy(['origin', 'path']);
// CTE: compute screen_view durations via window function (leadInFrame gives next event's timestamp)
const screenViewDurationsCte = clix(this.client, timezone)
.select([
'project_id',
'session_id',
'path',
'origin',
`dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration`,
])
.from(TABLE_NAMES.events, false)
.where('project_id', '=', projectId)
.where('name', '=', 'screen_view')
.where('path', '!=', '')
.where('created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
]);
// Pre-filtered sessions subquery for better performance
const sessionsSubquery = clix(this.client, timezone)
.select(['id', 'project_id', 'is_bounce'])
@@ -66,6 +84,7 @@ export class PagesService {
// Main query: aggregate events and calculate bounce rate from pre-filtered sessions
const query = clix(this.client, timezone)
.with('page_titles', titlesCte)
.with('screen_view_durations', screenViewDurationsCte)
.select<ITopPage>([
'e.origin as origin',
'e.path as path',
@@ -74,25 +93,18 @@ export class PagesService {
'count() as pageviews',
'round(avg(e.duration) / 1000 / 60, 2) as avg_duration',
`round(
(uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /
nullIf(uniq(e.session_id), 0),
(uniqIf(e.session_id, s.is_bounce = 1) * 100.0) /
nullIf(uniq(e.session_id), 0),
2
) as bounce_rate`,
])
.from(`${TABLE_NAMES.events} e`, false)
.from('screen_view_durations e', false)
.leftJoin(
sessionsSubquery,
'e.session_id = s.id AND e.project_id = s.project_id',
's'
)
.leftJoin('page_titles pt', 'concat(e.origin, e.path) = pt.page_key')
.where('e.project_id', '=', projectId)
.where('e.name', '=', 'screen_view')
.where('e.path', '!=', '')
.where('e.created_at', 'BETWEEN', [
clix.datetime(startDate, 'toDateTime'),
clix.datetime(endDate, 'toDateTime'),
])
.when(!!search, (q) => {
const term = `%${search}%`;
q.whereGroup()

9
packages/db/src/services/project.service.ts
View File

@@ -1,6 +1,6 @@
import { cacheable } from '@openpanel/redis';
import sqlstring from 'sqlstring';
import { TABLE_NAMES, chQuery } from '../clickhouse/client';
import { chQuery, TABLE_NAMES } from '../clickhouse/client';
import type { Prisma, Project } from '../prisma-client';
import { db } from '../prisma-client';
@@ -25,6 +25,7 @@ export async function getProjectById(id: string) {
return res;
}
/** L1 LRU (60s) + L2 Redis. clear() invalidates Redis + local LRU; other nodes may serve stale from LRU for up to 60s. */
export const getProjectByIdCached = cacheable(getProjectById, 60 * 60 * 24);
export async function getProjectWithClients(id: string) {
@@ -44,7 +45,7 @@ export async function getProjectWithClients(id: string) {
return res;
}
export async function getProjectsByOrganizationId(organizationId: string) {
export function getProjectsByOrganizationId(organizationId: string) {
return db.project.findMany({
where: {
organizationId,
@@ -95,7 +96,7 @@ export async function getProjects({
if (access.length > 0) {
return projects.filter((project) =>
access.some((a) => a.projectId === project.id),
access.some((a) => a.projectId === project.id)
);
}
@@ -104,7 +105,7 @@ export async function getProjects({
export const getProjectEventsCount = async (projectId: string) => {
const res = await chQuery<{ count: number }>(
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`,
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(projectId)} AND name NOT IN ('session_start', 'session_end')`
);
return res[0]?.count;
};

9
packages/db/src/services/salt.service.ts
View File

@@ -1,9 +1,9 @@
import { generateSalt } from '@openpanel/common/server';
import { cacheableLru } from '@openpanel/redis';
import { cacheable } from '@openpanel/redis';
import { db } from '../prisma-client';
export const getSalts = cacheableLru(
export const getSalts = cacheable(
'op:salt',
async () => {
const [curr, prev] = await db.salt.findMany({
@@ -24,10 +24,7 @@ export const getSalts = cacheableLru(
return salts;
},
{
maxSize: 2,
ttl: 60 * 5,
},
60 * 5,
);
export async function createInitialSalts() {