zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: use groupmq instead of bullmq for incoming events (#206)

Browse Source 
* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-04 21:07:55 +02:00
committed by GitHub
parent ca4a880acd
commit 0b4fcbad69
23 changed files with 1292 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/package.json
View File

@@ -38,6 +38,7 @@
"fastify": "^5.2.1",
"fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0",
"groupmq": "1.0.0-next.13",
"ico-to-png": "^0.2.2",
"jsonwebtoken": "^9.0.2",
"ramda": "^0.29.1",

104
apps/api/scripts/mock.ts
View File

@@ -3,6 +3,7 @@ import * as faker from '@faker-js/faker';
import { generateId } from '@openpanel/common';
import { hashPassword } from '@openpanel/common/server';
import { ClientType, db } from '@openpanel/db';
import { getRedisCache } from '@openpanel/redis';
import { v4 as uuidv4 } from 'uuid';
const DOMAIN_COUNT = 5;
@@ -260,6 +261,8 @@ function insertFakeEvents(events: Event[]) {
}
async function simultaneousRequests() {
await getRedisCache().flushdb();
await new Promise((resolve) => setTimeout(resolve, 1000));
const sessions: {
ip: string;
referrer: string;
@@ -272,9 +275,11 @@ async function simultaneousRequests() {
userAgent:
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
track: [
{ name: 'screen_view', path: '/home' },
{ name: 'button_click', element: 'signup' },
{ name: 'screen_view', path: '/pricing' },
{ name: 'screen_view', path: '/home', parallel: '1' },
{ name: 'button_click', element: 'signup', parallel: '1' },
{ name: 'article_viewed', articleId: '123', parallel: '1' },
{ name: 'screen_view', path: '/pricing', parallel: '1' },
{ name: 'screen_view', path: '/blog', parallel: '1' },
],
},
{
@@ -361,8 +366,9 @@ async function simultaneousRequests() {
{ name: 'screen_view', path: '/landing' },
{ name: 'screen_view', path: '/pricing' },
{ name: 'screen_view', path: '/blog' },
{ name: 'screen_view', path: '/blog/post-1' },
{ name: 'screen_view', path: '/blog/post-2' },
{ name: 'screen_view', path: '/blog/post-1', parallel: '1' },
{ name: 'screen_view', path: '/blog/post-2', parallel: '1' },
{ name: 'button_click', element: 'learn_more', parallel: '1' },
{ name: 'screen_view', path: '/blog/post-3' },
{ name: 'screen_view', path: '/blog/post-4' },
],
@@ -396,21 +402,85 @@ async function simultaneousRequests() {
};
for (const session of sessions) {
// Group tracks by parallel flag
const trackGroups: { parallel?: string; tracks: any[] }[] = [];
let currentGroup: { parallel?: string; tracks: any[] } = { tracks: [] };
for (const track of session.track) {
const { name, ...properties } = track;
screenView.track.payload.name = name ?? '';
screenView.track.payload.properties.__referrer = session.referrer ?? '';
if (name === 'screen_view') {
screenView.track.payload.properties.__path =
(screenView.headers.origin ?? '') + (properties.path ?? '');
if (track.parallel) {
// If this track has a parallel flag
if (currentGroup.parallel === track.parallel) {
// Same parallel group, add to current group
currentGroup.tracks.push(track);
} else {
// Different parallel group, finish current group and start new one
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
currentGroup = { parallel: track.parallel, tracks: [track] };
}
} else {
screenView.track.payload.name = track.name ?? '';
screenView.track.payload.properties = properties;
// No parallel flag, finish any parallel group and start individual track
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
currentGroup = { tracks: [track] };
}
screenView.headers['x-client-ip'] = session.ip;
screenView.headers['user-agent'] = session.userAgent;
await trackit(screenView);
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
}
// Add the last group
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
// Process each group
for (const group of trackGroups) {
if (group.parallel && group.tracks.length > 1) {
// Parallel execution for same-flagged tracks
console.log(
`Firing ${group.tracks.length} parallel requests with flag '${group.parallel}'`,
);
const promises = group.tracks.map(async (track) => {
const { name, parallel, ...properties } = track;
const event = JSON.parse(JSON.stringify(screenView));
event.track.payload.name = name ?? '';
event.track.payload.properties.__referrer = session.referrer ?? '';
if (name === 'screen_view') {
event.track.payload.properties.__path =
(event.headers.origin ?? '') + (properties.path ?? '');
} else {
event.track.payload.name = track.name ?? '';
event.track.payload.properties = properties;
}
event.headers['x-client-ip'] = session.ip;
event.headers['user-agent'] = session.userAgent;
return trackit(event);
});
await Promise.all(promises);
console.log(`Completed ${group.tracks.length} parallel requests`);
} else {
// Sequential execution for individual tracks
for (const track of group.tracks) {
const { name, parallel, ...properties } = track;
screenView.track.payload.name = name ?? '';
screenView.track.payload.properties.__referrer =
session.referrer ?? '';
if (name === 'screen_view') {
screenView.track.payload.properties.__path =
(screenView.headers.origin ?? '') + (properties.path ?? '');
} else {
screenView.track.payload.name = track.name ?? '';
screenView.track.payload.properties = properties;
}
screenView.headers['x-client-ip'] = session.ip;
screenView.headers['user-agent'] = session.userAgent;
await trackit(screenView);
}
}
// Add delay between groups (not within parallel groups)
// await new Promise((resolve) => setTimeout(resolve, Math.random() * 100));
}
}
}

65
apps/api/src/controllers/event.controller.ts
View File

@@ -3,8 +3,8 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { eventsQueue } from '@openpanel/queue';
import { getLock } from '@openpanel/redis';
import { eventsGroupQueue, eventsQueue } from '@openpanel/queue';
import { getLock, getRedisCache } from '@openpanel/redis';
import type { PostEventPayload } from '@openpanel/sdk';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
@@ -17,10 +17,14 @@ export async function postEvent(
}>,
reply: FastifyReply,
) {
const timestamp = getTimestamp(request.timestamp, request.body);
const { timestamp, isTimestampFromThePast } = getTimestamp(
request.timestamp,
request.body,
);
const ip = getClientIp(request)!;
const ua = request.headers['user-agent']!;
const projectId = request.client?.projectId;
const headers = getStringHeaders(request.headers);
if (!projectId) {
reply.status(400).send('missing origin');
@@ -56,31 +60,54 @@ export async function postEvent(
return;
}
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
const isGroupQueue = await getRedisCache().exists('group_queue');
if (isGroupQueue) {
const groupId = request.body?.profileId
? `${projectId}:${request.body?.profileId}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers: getStringHeaders(request.headers),
headers,
event: {
...request.body,
timestamp: timestamp.timestamp,
isTimestampFromThePast: timestamp.isTimestampFromThePast,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
groupId,
});
} else {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
projectId,
headers,
event: {
...request.body,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
},
);
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
},
},
);
}
reply.status(202).send('ok');
}

59
apps/api/src/controllers/track.controller.ts
View File

@@ -6,13 +6,14 @@ import { checkDuplicatedEvent } from '@/utils/deduplicate';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { eventsQueue } from '@openpanel/queue';
import { getLock } from '@openpanel/redis';
import { eventsGroupQueue, eventsQueue } from '@openpanel/queue';
import { getLock, getRedisCache } from '@openpanel/redis';
import type {
DecrementPayload,
IdentifyPayload,
IncrementPayload,
TrackHandlerPayload,
TrackPayload,
} from '@openpanel/sdk';
export function getStringHeaders(headers: FastifyRequest['headers']) {
@@ -260,11 +261,6 @@ export async function handler(
reply.status(200).send();
}
type TrackPayload = {
name: string;
properties?: Record<string, any>;
};
async function track({
payload,
currentDeviceId,
@@ -284,11 +280,14 @@ async function track({
timestamp: string;
isTimestampFromThePast: boolean;
}) {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
const isGroupQueue = await getRedisCache().exists('group_queue');
if (isGroupQueue) {
const groupId = payload.profileId
? `${projectId}:${payload.profileId}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
headers,
event: {
@@ -300,15 +299,35 @@ async function track({
currentDeviceId,
previousDeviceId,
},
},
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
groupId,
});
} else {
await eventsQueue.add(
'event',
{
type: 'incomingEvent',
payload: {
projectId,
headers,
event: {
...payload,
timestamp,
isTimestampFromThePast,
},
geo,
currentDeviceId,
previousDeviceId,
},
},
},
);
{
attempts: 3,
backoff: {
type: 'exponential',
delay: 200,
},
},
);
}
}
async function identify({