zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: use groupmq instead of bullmq for incoming events (#206)

Browse Source 
* wip

* wip working group queue

* wip

* wip

* wip

* fix: groupmq package (tests failed)

* minor fixes

* fix: zero is fine for duration

* add logger

* fix: make buffers more lightweight

* bump groupmq

* new buffers and bump groupmq

* fix: buffers based on comments

* fix: use profileId as groupId if exists

* bump groupmq

* add concurrency env for only events
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-04 21:07:55 +02:00
committed by GitHub
parent ca4a880acd
commit 0b4fcbad69
23 changed files with 1292 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
apps/api/scripts/mock.ts
View File

@@ -3,6 +3,7 @@ import * as faker from '@faker-js/faker';
import { generateId } from '@openpanel/common';
import { hashPassword } from '@openpanel/common/server';
import { ClientType, db } from '@openpanel/db';
import { getRedisCache } from '@openpanel/redis';
import { v4 as uuidv4 } from 'uuid';
const DOMAIN_COUNT = 5;
@@ -260,6 +261,8 @@ function insertFakeEvents(events: Event[]) {
}
async function simultaneousRequests() {
await getRedisCache().flushdb();
await new Promise((resolve) => setTimeout(resolve, 1000));
const sessions: {
ip: string;
referrer: string;
@@ -272,9 +275,11 @@ async function simultaneousRequests() {
userAgent:
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
track: [
{ name: 'screen_view', path: '/home' },
{ name: 'button_click', element: 'signup' },
{ name: 'screen_view', path: '/pricing' },
{ name: 'screen_view', path: '/home', parallel: '1' },
{ name: 'button_click', element: 'signup', parallel: '1' },
{ name: 'article_viewed', articleId: '123', parallel: '1' },
{ name: 'screen_view', path: '/pricing', parallel: '1' },
{ name: 'screen_view', path: '/blog', parallel: '1' },
],
},
{
@@ -361,8 +366,9 @@ async function simultaneousRequests() {
{ name: 'screen_view', path: '/landing' },
{ name: 'screen_view', path: '/pricing' },
{ name: 'screen_view', path: '/blog' },
{ name: 'screen_view', path: '/blog/post-1' },
{ name: 'screen_view', path: '/blog/post-2' },
{ name: 'screen_view', path: '/blog/post-1', parallel: '1' },
{ name: 'screen_view', path: '/blog/post-2', parallel: '1' },
{ name: 'button_click', element: 'learn_more', parallel: '1' },
{ name: 'screen_view', path: '/blog/post-3' },
{ name: 'screen_view', path: '/blog/post-4' },
],
@@ -396,21 +402,85 @@ async function simultaneousRequests() {
};
for (const session of sessions) {
// Group tracks by parallel flag
const trackGroups: { parallel?: string; tracks: any[] }[] = [];
let currentGroup: { parallel?: string; tracks: any[] } = { tracks: [] };
for (const track of session.track) {
const { name, ...properties } = track;
screenView.track.payload.name = name ?? '';
screenView.track.payload.properties.__referrer = session.referrer ?? '';
if (name === 'screen_view') {
screenView.track.payload.properties.__path =
(screenView.headers.origin ?? '') + (properties.path ?? '');
if (track.parallel) {
// If this track has a parallel flag
if (currentGroup.parallel === track.parallel) {
// Same parallel group, add to current group
currentGroup.tracks.push(track);
} else {
// Different parallel group, finish current group and start new one
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
currentGroup = { parallel: track.parallel, tracks: [track] };
}
} else {
screenView.track.payload.name = track.name ?? '';
screenView.track.payload.properties = properties;
// No parallel flag, finish any parallel group and start individual track
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
currentGroup = { tracks: [track] };
}
screenView.headers['x-client-ip'] = session.ip;
screenView.headers['user-agent'] = session.userAgent;
await trackit(screenView);
await new Promise((resolve) => setTimeout(resolve, Math.random() * 5000));
}
// Add the last group
if (currentGroup.tracks.length > 0) {
trackGroups.push(currentGroup);
}
// Process each group
for (const group of trackGroups) {
if (group.parallel && group.tracks.length > 1) {
// Parallel execution for same-flagged tracks
console.log(
`Firing ${group.tracks.length} parallel requests with flag '${group.parallel}'`,
);
const promises = group.tracks.map(async (track) => {
const { name, parallel, ...properties } = track;
const event = JSON.parse(JSON.stringify(screenView));
event.track.payload.name = name ?? '';
event.track.payload.properties.__referrer = session.referrer ?? '';
if (name === 'screen_view') {
event.track.payload.properties.__path =
(event.headers.origin ?? '') + (properties.path ?? '');
} else {
event.track.payload.name = track.name ?? '';
event.track.payload.properties = properties;
}
event.headers['x-client-ip'] = session.ip;
event.headers['user-agent'] = session.userAgent;
return trackit(event);
});
await Promise.all(promises);
console.log(`Completed ${group.tracks.length} parallel requests`);
} else {
// Sequential execution for individual tracks
for (const track of group.tracks) {
const { name, parallel, ...properties } = track;
screenView.track.payload.name = name ?? '';
screenView.track.payload.properties.__referrer =
session.referrer ?? '';
if (name === 'screen_view') {
screenView.track.payload.properties.__path =
(screenView.headers.origin ?? '') + (properties.path ?? '');
} else {
screenView.track.payload.name = track.name ?? '';
screenView.track.payload.properties = properties;
}
screenView.headers['x-client-ip'] = session.ip;
screenView.headers['user-agent'] = session.userAgent;
await trackit(screenView);
}
}
// Add delay between groups (not within parallel groups)
// await new Promise((resolve) => setTimeout(resolve, Math.random() * 100));
}
}
}