zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2026-03-11 23:28:20 +01:00
parent bc08566cd4
commit a672b73947
11 changed files with 196 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
packages/db/src/buffers/event-buffer.ts
View File

@@ -2,7 +2,6 @@ import { getSafeJson } from '@openpanel/json';
import {
type Redis,
getRedisCache,
getRedisPub,
publishEvent,
} from '@openpanel/redis';
import { ch } from '../clickhouse/client';
@@ -53,14 +52,10 @@ export class EventBuffer extends BaseBuffer {
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
private publishThrottleMs = process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS
? Number.parseInt(process.env.EVENT_BUFFER_PUBLISH_THROTTLE_MS, 10)
: 1000;
private lastPublishTime = 0;
private pendingPublishEvent: IClickhouseEvent | null = null;
private publishTimer: ReturnType<typeof setTimeout> | null = null;
private activeVisitorsExpiration = 60 * 5; // 5 minutes
/** How often (ms) we refresh the heartbeat key + zadd per visitor. */
private heartbeatRefreshMs = 60_000; // 1 minute
private lastHeartbeat = new Map<string, number>();
private queueKey = 'event_buffer:queue';
protected bufferCounterKey = 'event_buffer:total_count';
@@ -194,7 +189,7 @@ return added
}
}
add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {
add(event: IClickhouseEvent) {
const eventJson = JSON.stringify(event);
let type: PendingEvent['type'] = 'regular';
@@ -218,11 +213,6 @@ return added
type,
};
if (_multi) {
this.addToMulti(_multi, pendingEvent);
return;
}
this.pendingEvents.push(pendingEvent);
if (this.pendingEvents.length >= this.microBatchMaxSize) {
@@ -318,11 +308,7 @@ return added
await multi.exec();
this.flushRetryCount = 0;
const lastEvent = eventsToFlush[eventsToFlush.length - 1];
if (lastEvent) {
this.scheduleThrottledPublish(lastEvent.event);
}
this.pruneHeartbeatMap();
} catch (error) {
// Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -335,41 +321,13 @@ return added
});
} finally {
this.isFlushing = false;
}
}
private scheduleThrottledPublish(event: IClickhouseEvent) {
this.pendingPublishEvent = event;
const now = Date.now();
const timeSinceLastPublish = now - this.lastPublishTime;
if (timeSinceLastPublish >= this.publishThrottleMs) {
this.executeThrottledPublish();
return;
}
if (!this.publishTimer) {
const delay = this.publishThrottleMs - timeSinceLastPublish;
this.publishTimer = setTimeout(() => {
this.publishTimer = null;
this.executeThrottledPublish();
}, delay);
}
}
private executeThrottledPublish() {
if (!this.pendingPublishEvent) {
return;
}
const event = this.pendingPublishEvent;
this.pendingPublishEvent = null;
this.lastPublishTime = Date.now();
const result = publishEvent('events', 'received', transformEvent(event));
if (result instanceof Promise) {
result.catch(() => {});
// Events may have accumulated while we were flushing; schedule another flush if needed
if (this.pendingEvents.length > 0 && !this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flushLocalBuffer();
}, this.microBatchIntervalMs);
}
}
}
@@ -438,11 +396,13 @@ return added
});
}
const pubMulti = getRedisPub().multi();
const countByProject = new Map<string, number>();
for (const event of eventsToClickhouse) {
await publishEvent('events', 'saved', transformEvent(event), pubMulti);
countByProject.set(event.project_id, (countByProject.get(event.project_id) ?? 0) + 1);
}
for (const [projectId, count] of countByProject) {
publishEvent('events', 'batch', { projectId, count });
}
await pubMulti.exec();
await redis
.multi()
@@ -502,14 +462,34 @@ return added
});
}
private pruneHeartbeatMap() {
const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
for (const [key, ts] of this.lastHeartbeat) {
if (ts < cutoff) {
this.lastHeartbeat.delete(key);
}
}
}
private incrementActiveVisitorCount(
multi: ReturnType<Redis['multi']>,
projectId: string,
profileId: string,
) {
const key = `${projectId}:${profileId}`;
const now = Date.now();
const last = this.lastHeartbeat.get(key) ?? 0;
if (now - last < this.heartbeatRefreshMs) {
return;
}
this.lastHeartbeat.set(key, now);
const zsetKey = `live:visitors:${projectId}`;
return multi.zadd(zsetKey, now, profileId);
const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
multi
.zadd(zsetKey, now, profileId)
.set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
}
public async getActiveVisitorCount(projectId: string): Promise<number> {

3
packages/redis/publisher.ts
View File

@@ -10,8 +10,7 @@ export type IPublishChannels = {
};
};
events: {
received: IServiceEvent;
saved: IServiceEvent;
batch: { projectId: string; count: number };
};
notification: {
created: Prisma.NotificationUncheckedCreateInput;

50
packages/trpc/src/routers/realtime.ts
View File

@@ -1,18 +1,15 @@
import { z } from 'zod';
import {
type EventMeta,
TABLE_NAMES,
ch,
chQuery,
clix,
db,
formatClickhouseDate,
getEventList,
type IClickhouseEvent,
TABLE_NAMES,
transformEvent,
} from '@openpanel/db';
import { subMinutes } from 'date-fns';
import sqlstring from 'sqlstring';
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../trpc';
export const realtimeRouter = createTRPCRouter({
@@ -25,7 +22,7 @@ export const realtimeRouter = createTRPCRouter({
long: number;
lat: number;
}>(
`SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`,
`SELECT DISTINCT country, city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${sqlstring.escape(input.projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
);
return res;
@@ -33,25 +30,18 @@ export const realtimeRouter = createTRPCRouter({
activeSessions: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ input }) => {
return getEventList({
projectId: input.projectId,
take: 30,
select: {
name: true,
path: true,
origin: true,
referrer: true,
referrerName: true,
referrerType: true,
country: true,
device: true,
os: true,
browser: true,
createdAt: true,
profile: true,
meta: true,
},
});
const rows = await chQuery<IClickhouseEvent>(
`SELECT
name, session_id, created_at, path, origin, referrer, referrer_name,
country, city, region, os, os_version, browser, browser_version,
device
FROM ${TABLE_NAMES.events}
WHERE project_id = ${sqlstring.escape(input.projectId)}
AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}'
ORDER BY created_at DESC
LIMIT 50`
);
return rows.map(transformEvent);
}),
paths: protectedProcedure
.input(z.object({ projectId: z.string() }))
@@ -76,7 +66,7 @@ export const realtimeRouter = createTRPCRouter({
.where(
'created_at',
'>=',
formatClickhouseDate(subMinutes(new Date(), 30)),
formatClickhouseDate(subMinutes(new Date(), 30))
)
.groupBy(['path', 'origin'])
.orderBy('count', 'DESC')
@@ -106,7 +96,7 @@ export const realtimeRouter = createTRPCRouter({
.where(
'created_at',
'>=',
formatClickhouseDate(subMinutes(new Date(), 30)),
formatClickhouseDate(subMinutes(new Date(), 30))
)
.groupBy(['referrer_name'])
.orderBy('count', 'DESC')
@@ -137,7 +127,7 @@ export const realtimeRouter = createTRPCRouter({
.where(
'created_at',
'>=',
formatClickhouseDate(subMinutes(new Date(), 30)),
formatClickhouseDate(subMinutes(new Date(), 30))
)
.groupBy(['country', 'city'])
.orderBy('count', 'DESC')