zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: remove old event queue, cleaned up session handling, remove hacks

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-09 09:25:52 +02:00
parent a11f87dc3c
commit e7c21bc92c
16 changed files with 202 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/package.json
View File

@@ -38,7 +38,7 @@
"fastify": "^5.2.1",
"fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0",
"groupmq": "1.0.0-next.17",
"groupmq": "1.0.0-next.18",
"ico-to-png": "^0.2.2",
"jsonwebtoken": "^9.0.2",
"ramda": "^0.29.1",

74
apps/api/src/controllers/event.controller.ts
View File

@@ -3,8 +3,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { eventsGroupQueue, eventsQueue } from '@openpanel/queue';
import { getLock, getRedisCache } from '@openpanel/redis';
import { eventsGroupQueue } from '@openpanel/queue';
import type { PostEventPayload } from '@openpanel/sdk';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
@@ -61,57 +60,28 @@ export async function postEvent(
return;
}
const isGroupQueue = await getRedisCache().exists('group_queue');
if (isGroupQueue) {
const uaInfo = parseUserAgent(ua, request.body?.properties);
const groupId = uaInfo.isServer
? request.body?.profileId
? `${projectId}:${request.body?.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers,
event: {
...request.body,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
const uaInfo = parseUserAgent(ua, request.body?.properties);
const groupId = uaInfo.isServer
? request.body?.profileId
? `${projectId}:${request.body?.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers,
event: {
...request.body,
timestamp,
isTimestampFromThePast,
},
groupId,
});
} else {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
projectId,
headers,
event: {
...request.body,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
},
},
);
}
geo,
currentDeviceId,
previousDeviceId,
},
groupId,
});
reply.status(202).send('ok');
}

75
apps/api/src/controllers/track.controller.ts
View File

@@ -3,13 +3,11 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { path, assocPath, pathOr, pick } from 'ramda';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
import { logger } from '@/utils/logger';
import { generateId } from '@openpanel/common';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { eventsGroupQueue, eventsQueue } from '@openpanel/queue';
import { getLock, getRedisCache } from '@openpanel/redis';
import { eventsGroupQueue } from '@openpanel/queue';
import type {
DecrementPayload,
IdentifyPayload,
@@ -282,57 +280,28 @@ async function track({
timestamp: string;
isTimestampFromThePast: boolean;
}) {
const isGroupQueue = await getRedisCache().exists('group_queue');
if (isGroupQueue) {
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
const groupId = uaInfo.isServer
? payload.profileId
? `${projectId}:${payload.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers,
event: {
...payload,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
const groupId = uaInfo.isServer
? payload.profileId
? `${projectId}:${payload.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers,
event: {
...payload,
timestamp,
isTimestampFromThePast,
},
groupId,
});
} else {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
projectId,
headers,
event: {
...payload,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
},
},
);
}
geo,
currentDeviceId,
previousDeviceId,
},
groupId,
});
}
async function identify({

4
apps/api/src/utils/graceful-shutdown.ts
View File

@@ -1,7 +1,7 @@
import { ch, db } from '@openpanel/db';
import {
cronQueue,
eventsQueue,
eventsGroupQueue,
miscQueue,
notificationQueue,
sessionsQueue,
@@ -71,7 +71,7 @@ export async function shutdown(
// Step 6: Close Bull queues (graceful shutdown of queue state)
try {
await Promise.all([
eventsQueue.close(),
eventsGroupQueue.close(),
sessionsQueue.close(),
cronQueue.close(),
miscQueue.close(),

2
apps/worker/package.json
View File

@@ -23,7 +23,7 @@
"@openpanel/redis": "workspace:*",
"bullmq": "^5.8.7",
"express": "^4.18.2",
"groupmq": "1.0.0-next.17",
"groupmq": "1.0.0-next.18",
"prom-client": "^15.1.3",
"ramda": "^0.29.1",
"source-map-support": "^0.5.21",

3
apps/worker/src/boot-workers.ts
View File

@@ -5,7 +5,6 @@ import {
type EventsQueuePayloadIncomingEvent,
cronQueue,
eventsGroupQueue,
eventsQueue,
miscQueue,
notificationQueue,
queueLogger,
@@ -45,7 +44,6 @@ export async function bootWorkers() {
},
});
eventsGroupWorker.run();
const eventsWorker = new Worker(eventsQueue.name, eventsJob, workerOptions);
const sessionsWorker = new Worker(
sessionsQueue.name,
sessionsJob,
@@ -61,7 +59,6 @@ export async function bootWorkers() {
const workers = [
sessionsWorker,
eventsWorker,
cronWorker,
notificationWorker,
miscWorker,

2
apps/worker/src/index.ts
View File

@@ -7,7 +7,6 @@ import { createInitialSalts } from '@openpanel/db';
import {
cronQueue,
eventsGroupQueue,
eventsQueue,
miscQueue,
notificationQueue,
sessionsQueue,
@@ -36,7 +35,6 @@ async function start() {
createBullBoard({
queues: [
new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
new BullMQAdapter(eventsQueue),
new BullMQAdapter(sessionsQueue),
new BullMQAdapter(cronQueue),
new BullMQAdapter(notificationQueue),

129
apps/worker/src/jobs/events.create-session-end.ts
View File

@@ -3,6 +3,8 @@ import type { Job } from 'bullmq';
import { logger as baseLogger } from '@/utils/logger';
import { getTime } from '@openpanel/common';
import {
type IClickhouseSession,
type IServiceCreateEventPayload,
type IServiceEvent,
TABLE_NAMES,
checkNotificationRulesForSessionEnd,
@@ -10,37 +12,33 @@ import {
eventBuffer,
formatClickhouseDate,
getEvents,
getHasFunnelRules,
getNotificationRulesByProjectId,
sessionBuffer,
} from '@openpanel/db';
import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';
const MAX_SESSION_EVENTS = 500;
// Grabs session_start and screen_views + the last occured event
async function getNecessarySessionEvents({
async function getSessionEvents({
projectId,
sessionId,
createdAt,
startAt,
endAt,
}: {
projectId: string;
sessionId: string;
createdAt: Date;
startAt: Date;
endAt: Date;
}): Promise<ReturnType<typeof getEvents>> {
const sql = `
SELECT * FROM ${TABLE_NAMES.events}
WHERE
session_id = '${sessionId}'
AND project_id = '${projectId}'
AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'
AND (
name IN ('screen_view', 'session_start')
OR created_at = (
SELECT MAX(created_at)
FROM ${TABLE_NAMES.events}
WHERE session_id = '${sessionId}'
AND project_id = '${projectId}'
AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'
AND name NOT IN ('screen_view', 'session_start')
)
)
ORDER BY created_at DESC;
AND created_at BETWEEN '${formatClickhouseDate(startAt)}' AND '${formatClickhouseDate(endAt)}'
ORDER BY created_at DESC LIMIT ${MAX_SESSION_EVENTS};
`;
const [lastScreenView, eventsInDb] = await Promise.all([
@@ -63,62 +61,77 @@ async function getNecessarySessionEvents({
export async function createSessionEnd(
job: Job<EventsQueuePayloadCreateSessionEnd>,
) {
const { payload } = job.data;
const logger = baseLogger.child({
payload: job.data.payload,
payload,
jobId: job.id,
reqId: job.data.payload.properties?.__reqId ?? 'unknown',
reqId: payload.properties?.__reqId ?? 'unknown',
});
logger.info('Processing session end job');
const payload = job.data.payload;
const session = await sessionBuffer.getExistingSession(payload.sessionId);
const events = await getNecessarySessionEvents({
if (!session) {
throw new Error('Session not found');
}
try {
handleSessionEndNotifications({
session,
payload,
});
} catch (error) {
logger.error('Creating notificatios for session end failed', {
error,
});
}
const lastScreenView = await eventBuffer.getLastScreenView({
projectId: payload.projectId,
sessionId: payload.sessionId,
createdAt: payload.createdAt,
});
const sessionStart = events.find((event) => event.name === 'session_start');
const screenViews = events.filter((event) => event.name === 'screen_view');
const lastEvent = events[0];
if (!sessionStart) {
throw new Error('No session_start found');
}
if (!lastEvent) {
throw new Error('No last event found');
}
const sessionDuration =
lastEvent.createdAt.getTime() - sessionStart.createdAt.getTime();
await checkNotificationRulesForSessionEnd(events).catch(() => {
logger.error('Error checking notification rules for session end', {
data: job.data,
});
});
logger.info('Creating session_end event', {
sessionStart,
lastEvent,
screenViews,
sessionDuration,
events,
});
// Create session end event
return createEvent({
...sessionStart,
...payload,
properties: {
...sessionStart.properties,
...(screenViews[0]?.properties ?? {}),
__bounce: screenViews.length <= 1,
...payload.properties,
...(lastScreenView?.properties ?? {}),
__bounce: session.is_bounce,
},
name: 'session_end',
duration: sessionDuration,
path: screenViews[0]?.path ?? '',
createdAt: new Date(getTime(lastEvent.createdAt) + 1000),
profileId: lastEvent.profileId || sessionStart.profileId,
duration: session.duration ?? 0,
path: lastScreenView?.path ?? '',
createdAt: new Date(getTime(session.ended_at) + 1000),
profileId: lastScreenView?.profileId || payload.profileId,
});
}
async function handleSessionEndNotifications({
session,
payload,
}: {
session: IClickhouseSession;
payload: IServiceCreateEventPayload;
}) {
const notificationRules = await getNotificationRulesByProjectId(
payload.projectId,
);
const hasFunnelRules = getHasFunnelRules(notificationRules);
const isEventCountReasonable =
session.event_count + session.screen_view_count < MAX_SESSION_EVENTS;
if (hasFunnelRules && isEventCountReasonable) {
const events = await getSessionEvents({
projectId: payload.projectId,
sessionId: payload.sessionId,
startAt: new Date(session.created_at),
endAt: new Date(session.ended_at),
});
if (events.length > 0) {
await checkNotificationRulesForSessionEnd(events);
}
}
}

9
apps/worker/src/metrics.ts
View File

@@ -7,18 +7,13 @@ import {
profileBuffer,
sessionBuffer,
} from '@openpanel/db';
import {
cronQueue,
eventsGroupQueue,
eventsQueue,
sessionsQueue,
} from '@openpanel/queue';
import { cronQueue, eventsGroupQueue, sessionsQueue } from '@openpanel/queue';
const Registry = client.Registry;
export const register = new Registry();
const queues = [eventsQueue, sessionsQueue, cronQueue, eventsGroupQueue];
const queues = [sessionsQueue, cronQueue, eventsGroupQueue];
queues.forEach((queue) => {
register.registerMetric(

6
apps/worker/src/utils/session-handler.ts
View File

@@ -65,12 +65,6 @@ export async function getSessionEnd({
});
if (sessionEnd) {
// Hack: if session end job just got created, we want to give it a chance to complete
// So the order is correct
if (sessionEnd.job.timestamp > Date.now() - 50) {
await new Promise((resolve) => setTimeout(resolve, 100));
}
const existingSessionIsAnonymous =
sessionEnd.job.data.payload.profileId ===
sessionEnd.job.data.payload.deviceId;