zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: use groupmq instead of bullmq for incoming events (#206)

Browse Source 
* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-04 21:07:55 +02:00
committed by GitHub
parent ca4a880acd
commit 0b4fcbad69
23 changed files with 1292 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/package.json
View File

@@ -38,6 +38,7 @@
"fastify": "^5.2.1",
"fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0",
"groupmq": "1.0.0-next.13",
"ico-to-png": "^0.2.2",
"jsonwebtoken": "^9.0.2",
"ramda": "^0.29.1",

104
apps/api/scripts/mock.ts
View File

@@ -3,6 +3,7 @@ import * as faker from '@faker-js/faker';
import { generateId } from '@openpanel/common';
import { hashPassword } from '@openpanel/common/server';
import { ClientType, db } from '@openpanel/db';
import { getRedisCache } from '@openpanel/redis';
import { v4 as uuidv4 } from 'uuid';
const DOMAIN_COUNT = 5;
@@ -260,6 +261,8 @@ function insertFakeEvents(events: Event[]) {
}
async function simultaneousRequests() {
await getRedisCache().flushdb();
await new Promise((resolve) => setTimeout(resolve, 1000));
const sessions: {
ip: string;
referrer: string;
@@ -272,9 +275,11 @@ async function simultaneousRequests() {
userAgent:
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
track: [
{ name: 'screen_view', path: '/home' },
{ name: 'button_click', element: 'signup' },
{ name: 'screen_view', path: '/pricing' },
{ name: 'screen_view', path: '/home', parallel: '1' },
{ name: 'button_click', element: 'signup', parallel: '1' },
{ name: 'article_viewed', articleId: '123', parallel: '1' },
{ name: 'screen_view', path: '/pricing', parallel: '1' },
{ name: 'screen_view', path: '/blog', parallel: '1' },
],
},
{
@@ -361,8 +366,9 @@ async function simultaneousRequests() {
{ name: 'screen_view', path: '/landing' },
{ name: 'screen_view', path: '/pricing' },
{ name: 'screen_view', path: '/blog' },
{ name: 'screen_view', path: '/blog/post-1' },
{ name: 'screen_view', path: '/blog/post-2' },
{ name: 'screen_view', path: '/blog/post-1', parallel: '1' },
{ name: 'screen_view', path: '/blog/post-2', parallel: '1' },
{ name: 'button_click', element: 'learn_more', parallel: '1' },
{ name: 'screen_view', path: '/blog/post-3' },
{ name: 'screen_view', path: '/blog/post-4' },
],
@@ -396,21 +402,85 @@ async function simultaneousRequests() {
};
for (const session of sessions) {
// Group tracks by parallel flag
const trackGroups: { parallel?: string; tracks: any[] }[] = [];
let currentGroup: { parallel?: string; tracks: any[] } = { tracks: [] };
for (const track of session.track) {
const { name, ...properties } = track;
screenView.track.payload.name = name ?? '';
screenView.track.payload.properties.__referrer = session.referrer ?? '';
if (name === 'screen_view') {
screenView.track.payload.properties.__path =
(screenView.headers.origin ?? '') + (properties.path ?? '');
if (track.parallel) {
// If this track has a parallel flag
if (currentGroup.parallel === track.parallel) {
// Same parallel group, add to current group
currentGroup.tracks.push(track);
} else {
// Different parallel group, finish current group and start new one
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
currentGroup = { parallel: track.parallel, tracks: [track] };
}
} else {
screenView.track.payload.name = track.name ?? '';
screenView.track.payload.properties = properties;
// No parallel flag, finish any parallel group and start individual track
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
currentGroup = { tracks: [track] };
}
screenView.headers['x-client-ip'] = session.ip;
screenView.headers['user-agent'] = session.userAgent;
await trackit(screenView);
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
}
// Add the last group
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
// Process each group
for (const group of trackGroups) {
if (group.parallel && group.tracks.length > 1) {
// Parallel execution for same-flagged tracks
console.log(
`Firing ${group.tracks.length} parallel requests with flag '${group.parallel}'`,
);
const promises = group.tracks.map(async (track) => {
const { name, parallel, ...properties } = track;
const event = JSON.parse(JSON.stringify(screenView));
event.track.payload.name = name ?? '';
event.track.payload.properties.__referrer = session.referrer ?? '';
if (name === 'screen_view') {
event.track.payload.properties.__path =
(event.headers.origin ?? '') + (properties.path ?? '');
} else {
event.track.payload.name = track.name ?? '';
event.track.payload.properties = properties;
}
event.headers['x-client-ip'] = session.ip;
event.headers['user-agent'] = session.userAgent;
return trackit(event);
});
await Promise.all(promises);
console.log(`Completed ${group.tracks.length} parallel requests`);
} else {
// Sequential execution for individual tracks
for (const track of group.tracks) {
const { name, parallel, ...properties } = track;
screenView.track.payload.name = name ?? '';
screenView.track.payload.properties.__referrer =
session.referrer ?? '';
if (name === 'screen_view') {
screenView.track.payload.properties.__path =
(screenView.headers.origin ?? '') + (properties.path ?? '');
} else {
screenView.track.payload.name = track.name ?? '';
screenView.track.payload.properties = properties;
}
screenView.headers['x-client-ip'] = session.ip;
screenView.headers['user-agent'] = session.userAgent;
await trackit(screenView);
}
}
// Add delay between groups (not within parallel groups)
// await new Promise((resolve) => setTimeout(resolve, Math.random() * 100));
}
}
}

65
apps/api/src/controllers/event.controller.ts
View File

@@ -3,8 +3,8 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { eventsQueue } from '@openpanel/queue';
import { getLock } from '@openpanel/redis';
import { eventsGroupQueue, eventsQueue } from '@openpanel/queue';
import { getLock, getRedisCache } from '@openpanel/redis';
import type { PostEventPayload } from '@openpanel/sdk';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
@@ -17,10 +17,14 @@ export async function postEvent(
}>,
reply: FastifyReply,
) {
const timestamp = getTimestamp(request.timestamp, request.body);
const { timestamp, isTimestampFromThePast } = getTimestamp(
request.timestamp,
request.body,
);
const ip = getClientIp(request)!;
const ua = request.headers['user-agent']!;
const projectId = request.client?.projectId;
const headers = getStringHeaders(request.headers);
if (!projectId) {
reply.status(400).send('missing origin');
@@ -56,31 +60,54 @@ export async function postEvent(
return;
}
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
const isGroupQueue = await getRedisCache().exists('group_queue');
if (isGroupQueue) {
const groupId = request.body?.profileId
? `${projectId}:${request.body?.profileId}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers: getStringHeaders(request.headers),
headers,
event: {
...request.body,
timestamp: timestamp.timestamp,
isTimestampFromThePast: timestamp.isTimestampFromThePast,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
groupId,
});
} else {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
projectId,
headers,
event: {
...request.body,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
},
);
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
},
},
);
}
reply.status(202).send('ok');
}

59
apps/api/src/controllers/track.controller.ts
View File

@@ -6,13 +6,14 @@ import { checkDuplicatedEvent } from '@/utils/deduplicate';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { eventsQueue } from '@openpanel/queue';
import { getLock } from '@openpanel/redis';
import { eventsGroupQueue, eventsQueue } from '@openpanel/queue';
import { getLock, getRedisCache } from '@openpanel/redis';
import type {
DecrementPayload,
IdentifyPayload,
IncrementPayload,
TrackHandlerPayload,
TrackPayload,
} from '@openpanel/sdk';
export function getStringHeaders(headers: FastifyRequest['headers']) {
@@ -260,11 +261,6 @@ export async function handler(
reply.status(200).send();
}
type TrackPayload = {
name: string;
properties?: Record<string, any>;
};
async function track({
payload,
currentDeviceId,
@@ -284,11 +280,14 @@ async function track({
timestamp: string;
isTimestampFromThePast: boolean;
}) {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
const isGroupQueue = await getRedisCache().exists('group_queue');
if (isGroupQueue) {
const groupId = payload.profileId
? `${projectId}:${payload.profileId}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers,
event: {
@@ -300,15 +299,35 @@ async function track({
currentDeviceId,
previousDeviceId,
},
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
groupId,
});
} else {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
projectId,
headers,
event: {
...payload,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
},
);
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
},
},
);
}
}
async function identify({

3
apps/worker/package.json
View File

@@ -15,14 +15,15 @@
"@bull-board/express": "5.21.0",
"@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*",
"@openpanel/email": "workspace:*",
"@openpanel/integrations": "workspace:^",
"@openpanel/json": "workspace:*",
"@openpanel/logger": "workspace:*",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",
"@openpanel/email": "workspace:*",
"bullmq": "^5.8.7",
"express": "^4.18.2",
"groupmq": "1.0.0-next.13",
"prom-client": "^15.1.3",
"ramda": "^0.29.1",
"source-map-support": "^0.5.21",

35
apps/worker/src/boot-workers.ts
View File

@@ -2,18 +2,24 @@ import type { Queue, WorkerOptions } from 'bullmq';
import { Worker } from 'bullmq';
import {
type EventsQueuePayloadIncomingEvent,
cronQueue,
eventsGroupQueue,
eventsQueue,
miscQueue,
notificationQueue,
queueLogger,
sessionsQueue,
} from '@openpanel/queue';
import { getRedisQueue } from '@openpanel/redis';
import { performance } from 'node:perf_hooks';
import { setTimeout as sleep } from 'node:timers/promises';
import { Worker as GroupWorker } from 'groupmq';
import { cronJob } from './jobs/cron';
import { eventsJob } from './jobs/events';
import { incomingEventPure } from './jobs/events.incoming-event';
import { miscJob } from './jobs/misc';
import { notificationJob } from './jobs/notification';
import { sessionsJob } from './jobs/sessions';
@@ -21,10 +27,24 @@ import { logger } from './utils/logger';
const workerOptions: WorkerOptions = {
connection: getRedisQueue(),
concurrency: Number.parseInt(process.env.CONCURRENCY || '1', 10),
};
export async function bootWorkers() {
const eventsGroupWorker = new GroupWorker<
EventsQueuePayloadIncomingEvent['payload']
>({
concurrency: Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '1', 10),
logger: queueLogger,
queue: eventsGroupQueue,
handler: async (job) => {
logger.info('processing event (group queue)', {
groupId: job.groupId,
timestamp: job.data.event.timestamp,
});
await incomingEventPure(job.data);
},
});
eventsGroupWorker.run();
const eventsWorker = new Worker(eventsQueue.name, eventsJob, workerOptions);
const sessionsWorker = new Worker(
sessionsQueue.name,
@@ -45,29 +65,30 @@ export async function bootWorkers() {
cronWorker,
notificationWorker,
miscWorker,
eventsGroupWorker,
];
workers.forEach((worker) => {
worker.on('error', (error) => {
(worker as Worker).on('error', (error) => {
logger.error('worker error', {
worker: worker.name,
error,
});
});
worker.on('closed', () => {
(worker as Worker).on('closed', () => {
logger.info('worker closed', {
worker: worker.name,
});
});
worker.on('ready', () => {
(worker as Worker).on('ready', () => {
logger.info('worker ready', {
worker: worker.name,
});
});
worker.on('failed', (job) => {
(worker as Worker).on('failed', (job) => {
if (job) {
logger.error('job failed', {
worker: worker.name,
@@ -78,7 +99,7 @@ export async function bootWorkers() {
}
});
worker.on('completed', (job) => {
(worker as Worker).on('completed', (job) => {
if (job) {
logger.info('job completed', {
worker: worker.name,
@@ -91,7 +112,7 @@ export async function bootWorkers() {
}
});
worker.on('ioredis:close', () => {
(worker as Worker).on('ioredis:close', () => {
logger.error('worker closed due to ioredis:close', {
worker: worker.name,
});

3
apps/worker/src/index.ts
View File

@@ -6,6 +6,7 @@ import express from 'express';
import { createInitialSalts } from '@openpanel/db';
import {
cronQueue,
eventsGroupQueue,
eventsQueue,
miscQueue,
notificationQueue,
@@ -13,6 +14,7 @@ import {
} from '@openpanel/queue';
import client from 'prom-client';
import { BullBoardGroupMQAdapter } from 'groupmq';
import sourceMapSupport from 'source-map-support';
import { bootCron } from './boot-cron';
import { bootWorkers } from './boot-workers';
@@ -33,6 +35,7 @@ async function start() {
serverAdapter.setBasePath('/');
createBullBoard({
queues: [
new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
new BullMQAdapter(eventsQueue),
new BullMQAdapter(sessionsQueue),
new BullMQAdapter(cronQueue),

37
apps/worker/src/jobs/events.incoming-event.ts
View File

@@ -45,6 +45,14 @@ async function createEventAndNotify(
export async function incomingEvent(
job: Job<EventsQueuePayloadIncomingEvent>,
token?: string,
) {
return incomingEventPure(job.data.payload, job, token);
}
export async function incomingEventPure(
jobPayload: EventsQueuePayloadIncomingEvent['payload'],
job?: Job<EventsQueuePayloadIncomingEvent>,
token?: string,
) {
const {
geo,
@@ -53,7 +61,7 @@ export async function incomingEvent(
projectId,
currentDeviceId,
previousDeviceId,
} = job.data.payload;
} = jobPayload;
const properties = body.properties ?? {};
const reqId = headers['request-id'] ?? 'unknown';
const logger = baseLogger.child({
@@ -151,11 +159,7 @@ export async function incomingEvent(
origin: screenView?.origin ?? baseEvent.origin,
};
return createEventAndNotify(
payload as IServiceEvent,
job.data.payload,
logger,
);
return createEventAndNotify(payload as IServiceEvent, jobPayload, logger);
}
const sessionEnd = await getSessionEnd({
@@ -186,21 +190,22 @@ export async function incomingEvent(
if (!sessionEnd) {
// Too avoid several created sessions we just throw if a lock exists
// This will than retry the job
const lock = await getLock(
`create-session-end:${currentDeviceId}`,
'locked',
1000,
);
if (job) {
const lock = await getLock(
`create-session-end:${currentDeviceId}`,
'locked',
1000,
);
if (!lock) {
logger.warn('Move incoming event to delayed');
await job.moveToDelayed(Date.now() + 50, token);
throw new DelayedError();
if (!lock) {
await job.moveToDelayed(Date.now() + 50, token);
throw new DelayedError();
}
}
await createSessionStart({ payload });
}
const event = await createEventAndNotify(payload, job.data.payload, logger);
const event = await createEventAndNotify(payload, jobPayload, logger);
if (!sessionEnd) {
await createSessionEndJob({ payload });

9
apps/worker/src/metrics.ts
View File

@@ -7,13 +7,18 @@ import {
profileBuffer,
sessionBuffer,
} from '@openpanel/db';
import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';
import {
cronQueue,
eventsGroupQueue,
eventsQueue,
sessionsQueue,
} from '@openpanel/queue';
const Registry = client.Registry;
export const register = new Registry();
const queues = [eventsQueue, sessionsQueue, cronQueue];
const queues = [eventsQueue, sessionsQueue, cronQueue, eventsGroupQueue];
queues.forEach((queue) => {
register.registerMetric(