zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: overall perf improvements

Browse Source 
* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-15 22:13:59 +01:00
committed by GitHub
parent 38cc53890a
commit da59622dce
66 changed files with 5042 additions and 3860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
apps/worker/src/boot-cron.ts
View File

@@ -44,43 +44,30 @@ export async function bootCron() {
});
}
// Add repeatable jobs
for (const job of jobs) {
await cronQueue.add(
job.name,
{
type: job.type,
payload: undefined,
},
{
jobId: job.type,
repeat:
typeof job.pattern === 'number'
? {
every: job.pattern,
}
: {
pattern: job.pattern,
},
},
);
logger.info('Updating cron jobs');
const jobSchedulers = await cronQueue.getJobSchedulers();
for (const jobScheduler of jobSchedulers) {
await cronQueue.removeJobScheduler(jobScheduler.key);
}
// Remove outdated repeatable jobs
const repeatableJobs = await cronQueue.getRepeatableJobs();
for (const repeatableJob of repeatableJobs) {
const match = jobs.find(
(job) => `${job.name}:${job.type}:::${job.pattern}` === repeatableJob.key,
// Add repeatable jobs
for (const job of jobs) {
await cronQueue.upsertJobScheduler(
job.type,
typeof job.pattern === 'number'
? {
every: job.pattern,
}
: {
pattern: job.pattern,
},
{
data: {
type: job.type,
payload: undefined,
},
},
);
if (match) {
logger.info('Repeatable job exists', {
key: repeatableJob.key,
});
} else {
logger.info('Removing repeatable job', {
key: repeatableJob.key,
});
cronQueue.removeRepeatableByKey(repeatableJob.key);
}
}
}

253
apps/worker/src/boot-workers.ts
View File

@@ -2,9 +2,10 @@ import type { Queue, WorkerOptions } from 'bullmq';
import { Worker } from 'bullmq';
import {
EVENTS_GROUP_QUEUES_SHARDS,
type EventsQueuePayloadIncomingEvent,
cronQueue,
eventsGroupQueue,
eventsGroupQueues,
importQueue,
miscQueue,
notificationQueue,
@@ -18,59 +19,179 @@ import { setTimeout as sleep } from 'node:timers/promises';
import { Worker as GroupWorker } from 'groupmq';
import { cronJob } from './jobs/cron';
import { eventsJob } from './jobs/events';
import { incomingEventPure } from './jobs/events.incoming-event';
import { incomingEvent } from './jobs/events.incoming-event';
import { importJob } from './jobs/import';
import { miscJob } from './jobs/misc';
import { notificationJob } from './jobs/notification';
import { sessionsJob } from './jobs/sessions';
import { eventsGroupJobDuration } from './metrics';
import { logger } from './utils/logger';
const workerOptions: WorkerOptions = {
connection: getRedisQueue(),
};
export async function bootWorkers() {
const eventsGroupWorker = new GroupWorker<
EventsQueuePayloadIncomingEvent['payload']
>({
concurrency: Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '1', 10),
logger: queueLogger,
queue: eventsGroupQueue,
handler: async (job) => {
logger.info('processing event (group queue)', {
groupId: job.groupId,
timestamp: job.data.event.timestamp,
});
await incomingEventPure(job.data);
},
});
eventsGroupWorker.run();
const sessionsWorker = new Worker(
sessionsQueue.name,
sessionsJob,
workerOptions,
);
const cronWorker = new Worker(cronQueue.name, cronJob, workerOptions);
const notificationWorker = new Worker(
notificationQueue.name,
notificationJob,
workerOptions,
);
const miscWorker = new Worker(miscQueue.name, miscJob, workerOptions);
const importWorker = new Worker(importQueue.name, importJob, {
...workerOptions,
concurrency: Number.parseInt(process.env.IMPORT_JOB_CONCURRENCY || '1', 10),
});
type QueueName = string; // Can be: events, events_N (where N is 0 to shards-1), sessions, cron, notification, misc
const workers = [
sessionsWorker,
cronWorker,
notificationWorker,
miscWorker,
importWorker,
// eventsGroupWorker,
];
/**
* Parses the ENABLED_QUEUES environment variable and returns an array of queue names to start.
* If no env var is provided, returns all queues.
*
* Supported queue names:
* - events - All event shards (events_0, events_1, ..., events_N)
* - events_N - Individual event shard (where N is 0 to EVENTS_GROUP_QUEUES_SHARDS-1)
* - sessions, cron, notification, misc
*/
function getEnabledQueues(): QueueName[] {
const enabledQueuesEnv = process.env.ENABLED_QUEUES?.trim();
if (!enabledQueuesEnv) {
logger.info('No ENABLED_QUEUES specified, starting all queues', {
totalEventShards: EVENTS_GROUP_QUEUES_SHARDS,
});
return ['events', 'sessions', 'cron', 'notification', 'misc', 'import'];
}
const queues = enabledQueuesEnv
.split(',')
.map((q) => q.trim())
.filter(Boolean);
logger.info('Starting queues from ENABLED_QUEUES', {
queues,
totalEventShards: EVENTS_GROUP_QUEUES_SHARDS,
});
return queues;
}
/**
* Gets the concurrency setting for a queue from environment variables.
* Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., EVENTS_0_CONCURRENCY=32)
*/
function getConcurrencyFor(queueName: string, defaultValue = 1): number {
const envKey = `${queueName.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_CONCURRENCY`;
const value = process.env[envKey];
if (value) {
const parsed = Number.parseInt(value, 10);
if (!Number.isNaN(parsed) && parsed > 0) {
return parsed;
}
}
return defaultValue;
}
export async function bootWorkers() {
const enabledQueues = getEnabledQueues();
const workers: (Worker | GroupWorker<any>)[] = [];
// Start event workers based on enabled queues
const eventQueuesToStart: number[] = [];
if (enabledQueues.includes('events')) {
// Start all event shards
for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
eventQueuesToStart.push(i);
}
} else {
// Start specific event shards (events_0, events_1, etc.)
for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
if (enabledQueues.includes(`events_${i}`)) {
eventQueuesToStart.push(i);
}
}
}
for (const index of eventQueuesToStart) {
const queue = eventsGroupQueues[index];
if (!queue) continue;
const queueName = `events_${index}`;
const concurrency = getConcurrencyFor(
queueName,
Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10),
);
const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
queue,
concurrency,
logger: queueLogger,
blockingTimeoutSec: Number.parseFloat(
process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1',
),
handler: async (job) => {
return await incomingEvent(job.data);
},
});
worker.run();
workers.push(worker);
logger.info(`Started worker for ${queueName}`, { concurrency });
}
// Start sessions worker
if (enabledQueues.includes('sessions')) {
const concurrency = getConcurrencyFor('sessions');
const sessionsWorker = new Worker(sessionsQueue.name, sessionsJob, {
...workerOptions,
concurrency,
});
workers.push(sessionsWorker);
logger.info('Started worker for sessions', { concurrency });
}
// Start cron worker
if (enabledQueues.includes('cron')) {
const concurrency = getConcurrencyFor('cron');
const cronWorker = new Worker(cronQueue.name, cronJob, {
...workerOptions,
concurrency,
});
workers.push(cronWorker);
logger.info('Started worker for cron', { concurrency });
}
// Start notification worker
if (enabledQueues.includes('notification')) {
const concurrency = getConcurrencyFor('notification');
const notificationWorker = new Worker(
notificationQueue.name,
notificationJob,
{ ...workerOptions, concurrency },
);
workers.push(notificationWorker);
logger.info('Started worker for notification', { concurrency });
}
// Start misc worker
if (enabledQueues.includes('misc')) {
const concurrency = getConcurrencyFor('misc');
const miscWorker = new Worker(miscQueue.name, miscJob, {
...workerOptions,
concurrency,
});
workers.push(miscWorker);
logger.info('Started worker for misc', { concurrency });
}
// Start import worker
if (enabledQueues.includes('import')) {
const concurrency = getConcurrencyFor('import');
const importWorker = new Worker(importQueue.name, importJob, {
...workerOptions,
concurrency,
});
workers.push(importWorker);
logger.info('Started worker for import', { concurrency });
}
if (workers.length === 0) {
logger.warn(
'No workers started. Check ENABLED_QUEUES environment variable.',
);
}
workers.forEach((worker) => {
(worker as Worker).on('error', (error) => {
@@ -94,6 +215,13 @@ export async function bootWorkers() {
(worker as Worker).on('failed', (job) => {
if (job) {
if (job.processedOn && job.finishedOn) {
const elapsed = job.finishedOn - job.processedOn;
eventsGroupJobDuration.observe(
{ name: worker.name, status: 'failed' },
elapsed,
);
}
logger.error('job failed', {
jobId: job.id,
worker: worker.name,
@@ -106,15 +234,18 @@ export async function bootWorkers() {
(worker as Worker).on('completed', (job) => {
if (job) {
logger.info('job completed', {
jobId: job.id,
worker: worker.name,
data: job.data,
elapsed:
job.processedOn && job.finishedOn
? job.finishedOn - job.processedOn
: undefined,
});
if (job.processedOn && job.finishedOn) {
const elapsed = job.finishedOn - job.processedOn;
logger.info('job completed', {
jobId: job.id,
worker: worker.name,
elapsed,
});
eventsGroupJobDuration.observe(
{ name: worker.name, status: 'success' },
elapsed,
);
}
}
});
@@ -135,8 +266,14 @@ export async function bootWorkers() {
});
try {
const time = performance.now();
await waitForQueueToEmpty(cronQueue);
// Wait for cron queue to empty if it's running
if (enabledQueues.includes('cron')) {
await waitForQueueToEmpty(cronQueue);
}
await Promise.all(workers.map((worker) => worker.close()));
logger.info('workers closed successfully', {
elapsed: performance.now() - time,
});
@@ -155,15 +292,7 @@ export async function bootWorkers() {
['uncaughtException', 'unhandledRejection', 'SIGTERM', 'SIGINT'].forEach(
(evt) => {
process.on(evt, (code) => {
if (process.env.NODE_ENV === 'production') {
exitHandler(evt, code);
} else {
logger.info('Shutting down for development', {
event: evt,
code,
});
process.exit(0);
}
exitHandler(evt, code);
});
},
);

6
apps/worker/src/index.ts
View File

@@ -4,7 +4,7 @@ import { ExpressAdapter } from '@bull-board/express';
import { createInitialSalts } from '@openpanel/db';
import {
cronQueue,
eventsGroupQueue,
eventsGroupQueues,
importQueue,
miscQueue,
notificationQueue,
@@ -34,7 +34,9 @@ async function start() {
serverAdapter.setBasePath('/');
createBullBoard({
queues: [
new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
...eventsGroupQueues.map(
(queue) => new BullBoardGroupMQAdapter(queue) as any,
),
new BullMQAdapter(sessionsQueue),
new BullMQAdapter(cronQueue),
new BullMQAdapter(notificationQueue),

11
apps/worker/src/jobs/events.create-session-end.ts
View File

@@ -1,13 +1,13 @@
import type { Job } from 'bullmq';
import { logger as baseLogger } from '@/utils/logger';
import { getTime } from '@openpanel/common';
import {
type IClickhouseSession,
type IServiceCreateEventPayload,
type IServiceEvent,
TABLE_NAMES,
checkNotificationRulesForSessionEnd,
convertClickhouseDateToJs,
createEvent,
eventBuffer,
formatClickhouseDate,
@@ -65,10 +65,9 @@ export async function createSessionEnd(
const logger = baseLogger.child({
payload,
jobId: job.id,
reqId: payload.properties?.__reqId ?? 'unknown',
});
logger.info('Processing session end job');
logger.debug('Processing session end job');
const session = await sessionBuffer.getExistingSession(payload.sessionId);
@@ -77,7 +76,7 @@ export async function createSessionEnd(
}
try {
handleSessionEndNotifications({
await handleSessionEndNotifications({
session,
payload,
});
@@ -103,7 +102,9 @@ export async function createSessionEnd(
name: 'session_end',
duration: session.duration ?? 0,
path: lastScreenView?.path ?? '',
createdAt: new Date(getTime(session.ended_at) + 1000),
createdAt: new Date(
convertClickhouseDateToJs(session.ended_at).getTime() + 100,
),
profileId: lastScreenView?.profileId || payload.profileId,
});
}

26
apps/worker/src/jobs/events.incoming-event.ts
View File

@@ -18,9 +18,7 @@ import {
} from '@openpanel/db';
import type { ILogger } from '@openpanel/logger';
import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
import type { Job } from 'bullmq';
import * as R from 'ramda';
import { omit } from 'ramda';
import { v4 as uuid } from 'uuid';
const GLOBAL_PROPERTIES = ['__path', '__referrer'];
@@ -33,10 +31,9 @@ const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>
async function createEventAndNotify(
payload: IServiceCreateEventPayload,
jobData: Job<EventsQueuePayloadIncomingEvent>['data']['payload'],
logger: ILogger,
) {
logger.info('Creating event', { event: payload, jobData });
logger.info('Creating event', { event: payload });
const [event] = await Promise.all([
createEvent(payload),
checkNotificationRulesForEvent(payload).catch(() => {}),
@@ -45,16 +42,7 @@ async function createEventAndNotify(
}
export async function incomingEvent(
job: Job<EventsQueuePayloadIncomingEvent>,
token?: string,
) {
return incomingEventPure(job.data.payload, job, token);
}
export async function incomingEventPure(
jobPayload: EventsQueuePayloadIncomingEvent['payload'],
job?: Job<EventsQueuePayloadIncomingEvent>,
token?: string,
) {
const {
geo,
@@ -63,6 +51,7 @@ export async function incomingEventPure(
projectId,
currentDeviceId,
previousDeviceId,
uaInfo: _uaInfo,
} = jobPayload;
const properties = body.properties ?? {};
const reqId = headers['request-id'] ?? 'unknown';
@@ -93,18 +82,17 @@ export async function incomingEventPure(
const userAgent = headers['user-agent'];
const sdkName = headers['openpanel-sdk-name'];
const sdkVersion = headers['openpanel-sdk-version'];
const uaInfo = parseUserAgent(userAgent, properties);
// TODO: Remove both user-agent and parseUserAgent
const uaInfo = _uaInfo ?? parseUserAgent(userAgent, properties);
const baseEvent = {
name: body.name,
profileId,
projectId,
properties: omit(GLOBAL_PROPERTIES, {
properties: R.omit(GLOBAL_PROPERTIES, {
...properties,
__user_agent: userAgent,
__hash: hash,
__query: query,
__reqId: reqId,
}),
createdAt,
duration: 0,
@@ -161,7 +149,7 @@ export async function incomingEventPure(
origin: screenView?.origin ?? baseEvent.origin,
};
return createEventAndNotify(payload as IServiceEvent, jobPayload, logger);
return createEventAndNotify(payload as IServiceEvent, logger);
}
const sessionEnd = await getSessionEnd({
@@ -197,7 +185,7 @@ export async function incomingEventPure(
});
}
const event = await createEventAndNotify(payload, jobPayload, logger);
const event = await createEventAndNotify(payload, logger);
if (!sessionEnd) {
logger.info('Creating session end job', { event: payload });

211
apps/worker/src/jobs/events.incoming-events.test.ts
View File

@@ -1,6 +1,9 @@
import { type IServiceEvent, createEvent } from '@openpanel/db';
import { eventBuffer } from '@openpanel/db';
import { sessionsQueue } from '@openpanel/queue';
import {
type EventsQueuePayloadIncomingEvent,
sessionsQueue,
} from '@openpanel/queue';
import type { Job } from 'bullmq';
import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest';
import { incomingEvent } from './events.incoming-event';
@@ -32,6 +35,28 @@ const geo = {
latitude: 0,
};
const uaInfo: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
isServer: false,
device: 'desktop',
os: 'Windows',
osVersion: '10',
browser: 'Chrome',
browserVersion: '91.0.4472.124',
brand: '',
model: '',
};
const uaInfoServer: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
isServer: true,
device: 'server',
os: '',
osVersion: '',
browser: '',
browserVersion: '',
brand: '',
model: '',
};
describe('incomingEvent', () => {
beforeEach(() => {
vi.clearAllMocks();
@@ -41,31 +66,29 @@ describe('incomingEvent', () => {
const spySessionsQueueAdd = vi.spyOn(sessionsQueue, 'add');
const timestamp = new Date();
// Mock job data
const jobData = {
payload: {
geo,
event: {
name: 'test_event',
timestamp: timestamp.toISOString(),
properties: { __path: 'https://example.com/test' },
},
headers: {
'request-id': '123',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'openpanel-sdk-name': 'web',
'openpanel-sdk-version': '1.0.0',
},
projectId,
currentDeviceId,
previousDeviceId,
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
geo,
event: {
name: 'test_event',
timestamp: timestamp.toISOString(),
isTimestampFromThePast: false,
properties: { __path: 'https://example.com/test' },
},
uaInfo,
headers: {
'request-id': '123',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'openpanel-sdk-name': 'web',
'openpanel-sdk-version': '1.0.0',
},
projectId,
currentDeviceId,
previousDeviceId,
};
const job = { data: jobData } as Job;
// Execute the job
await incomingEvent(job);
await incomingEvent(jobData);
const event = {
name: 'test_event',
@@ -78,8 +101,6 @@ describe('incomingEvent', () => {
properties: {
__hash: undefined,
__query: undefined,
__user_agent: jobData.payload.headers['user-agent'],
__reqId: jobData.payload.headers['request-id'],
},
createdAt: timestamp,
country: 'US',
@@ -92,16 +113,16 @@ describe('incomingEvent', () => {
browser: 'Chrome',
browserVersion: '91.0.4472.124',
device: 'desktop',
brand: undefined,
model: undefined,
brand: '',
model: '',
duration: 0,
path: '/test',
origin: 'https://example.com',
referrer: '',
referrerName: '',
referrerType: '',
sdkName: jobData.payload.headers['openpanel-sdk-name'],
sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
sdkName: jobData.headers['openpanel-sdk-name'],
sdkVersion: jobData.headers['openpanel-sdk-version'],
};
expect(spySessionsQueueAdd).toHaveBeenCalledWith(
@@ -135,29 +156,27 @@ describe('incomingEvent', () => {
const timestamp = new Date();
// Mock job data
const jobData = {
payload: {
geo,
event: {
name: 'test_event',
timestamp: timestamp.toISOString(),
properties: { __path: 'https://example.com/test' },
},
headers: {
'request-id': '123',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'openpanel-sdk-name': 'web',
'openpanel-sdk-version': '1.0.0',
},
projectId,
currentDeviceId,
previousDeviceId,
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
geo,
event: {
name: 'test_event',
timestamp: timestamp.toISOString(),
properties: { __path: 'https://example.com/test' },
isTimestampFromThePast: false,
},
headers: {
'request-id': '123',
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'openpanel-sdk-name': 'web',
'openpanel-sdk-version': '1.0.0',
},
uaInfo,
projectId,
currentDeviceId,
previousDeviceId,
};
const job = { data: jobData } as Job;
const changeDelay = vi.fn();
const updateData = vi.fn();
spySessionsQueueGetJob.mockResolvedValueOnce({
@@ -175,7 +194,7 @@ describe('incomingEvent', () => {
},
} as Partial<Job> as Job);
// Execute the job
await incomingEvent(job);
await incomingEvent(jobData);
const event = {
name: 'test_event',
@@ -186,8 +205,6 @@ describe('incomingEvent', () => {
properties: {
__hash: undefined,
__query: undefined,
__user_agent: jobData.payload.headers['user-agent'],
__reqId: jobData.payload.headers['request-id'],
},
createdAt: timestamp,
country: 'US',
@@ -200,16 +217,16 @@ describe('incomingEvent', () => {
browser: 'Chrome',
browserVersion: '91.0.4472.124',
device: 'desktop',
brand: undefined,
model: undefined,
brand: '',
model: '',
duration: 0,
path: '/test',
origin: 'https://example.com',
referrer: '',
referrerName: '',
referrerType: '',
sdkName: jobData.payload.headers['openpanel-sdk-name'],
sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
sdkName: jobData.headers['openpanel-sdk-name'],
sdkVersion: jobData.headers['openpanel-sdk-version'],
};
expect(spySessionsQueueAdd).toHaveBeenCalledTimes(0);
@@ -220,29 +237,27 @@ describe('incomingEvent', () => {
it('should handle server events (with existing screen view)', async () => {
const timestamp = new Date();
const jobData = {
payload: {
geo,
event: {
name: 'server_event',
timestamp: timestamp.toISOString(),
properties: { custom_property: 'test_value' },
profileId: 'profile-123',
},
headers: {
'user-agent': 'OpenPanel Server/1.0',
'openpanel-sdk-name': 'server',
'openpanel-sdk-version': '1.0.0',
'request-id': '123',
},
projectId,
currentDeviceId: '',
previousDeviceId: '',
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
geo,
event: {
name: 'server_event',
timestamp: timestamp.toISOString(),
properties: { custom_property: 'test_value' },
profileId: 'profile-123',
isTimestampFromThePast: false,
},
headers: {
'user-agent': 'OpenPanel Server/1.0',
'openpanel-sdk-name': 'server',
'openpanel-sdk-version': '1.0.0',
'request-id': '123',
},
projectId,
currentDeviceId: '',
previousDeviceId: '',
uaInfo: uaInfoServer,
};
const job = { data: jobData } as Job;
const mockLastScreenView = {
deviceId: 'last-device-123',
sessionId: 'last-session-456',
@@ -268,7 +283,7 @@ describe('incomingEvent', () => {
mockLastScreenView as IServiceEvent,
);
await incomingEvent(job);
await incomingEvent(jobData);
expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
name: 'server_event',
@@ -278,8 +293,6 @@ describe('incomingEvent', () => {
projectId,
properties: {
custom_property: 'test_value',
__user_agent: 'OpenPanel Server/1.0',
__reqId: '123',
__hash: undefined,
__query: undefined,
},
@@ -311,33 +324,31 @@ describe('incomingEvent', () => {
it('should handle server events (without existing screen view)', async () => {
const timestamp = new Date();
const jobData = {
payload: {
geo,
event: {
name: 'server_event',
timestamp: timestamp.toISOString(),
properties: { custom_property: 'test_value' },
profileId: 'profile-123',
},
headers: {
'user-agent': 'OpenPanel Server/1.0',
'openpanel-sdk-name': 'server',
'openpanel-sdk-version': '1.0.0',
'request-id': '123',
},
projectId,
currentDeviceId: '',
previousDeviceId: '',
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
geo,
event: {
name: 'server_event',
timestamp: timestamp.toISOString(),
properties: { custom_property: 'test_value' },
profileId: 'profile-123',
isTimestampFromThePast: false,
},
headers: {
'user-agent': 'OpenPanel Server/1.0',
'openpanel-sdk-name': 'server',
'openpanel-sdk-version': '1.0.0',
'request-id': '123',
},
projectId,
currentDeviceId: '',
previousDeviceId: '',
uaInfo: uaInfoServer,
};
const job = { data: jobData } as Job;
// Mock getLastScreenView to return null
vi.mocked(eventBuffer.getLastScreenView).mockResolvedValueOnce(null);
await incomingEvent(job);
await incomingEvent(jobData);
expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
name: 'server_event',
@@ -347,8 +358,6 @@ describe('incomingEvent', () => {
projectId,
properties: {
custom_property: 'test_value',
__user_agent: 'OpenPanel Server/1.0',
__reqId: '123',
__hash: undefined,
__query: undefined,
},

15
apps/worker/src/jobs/events.ts
View File

@@ -1,15 +0,0 @@
import type { Job } from 'bullmq';
import type {
EventsQueuePayload,
EventsQueuePayloadIncomingEvent,
} from '@openpanel/queue';
import { incomingEvent } from './events.incoming-event';
export async function eventsJob(job: Job<EventsQueuePayload>, token?: string) {
return await incomingEvent(
job as Job<EventsQueuePayloadIncomingEvent>,
token,
);
}

25
apps/worker/src/metrics.ts
View File

@@ -2,23 +2,32 @@ import client from 'prom-client';
import {
botBuffer,
db,
eventBuffer,
profileBuffer,
sessionBuffer,
} from '@openpanel/db';
import { cronQueue, eventsGroupQueue, sessionsQueue } from '@openpanel/queue';
import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
const Registry = client.Registry;
export const register = new Registry();
const queues = [sessionsQueue, cronQueue, eventsGroupQueue];
const queues = [sessionsQueue, cronQueue, ...eventsGroupQueues];
// Histogram to track job processing time for eventsGroupQueues
export const eventsGroupJobDuration = new client.Histogram({
name: 'job_duration_ms',
help: 'Duration of job processing (in ms)',
labelNames: ['name', 'status'],
buckets: [10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10000, 30000], // 10ms to 30s
});
register.registerMetric(eventsGroupJobDuration);
queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_active_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_active_count`,
help: 'Active count',
async collect() {
const metric = await queue.getActiveCount();
@@ -29,7 +38,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_delayed_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_delayed_count`,
help: 'Delayed count',
async collect() {
const metric = await queue.getDelayedCount();
@@ -40,7 +49,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_failed_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_failed_count`,
help: 'Failed count',
async collect() {
const metric = await queue.getFailedCount();
@@ -51,7 +60,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_completed_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_completed_count`,
help: 'Completed count',
async collect() {
const metric = await queue.getCompletedCount();
@@ -62,7 +71,7 @@ queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
name: `${queue.name}_waiting_count`,
name: `${queue.name.replace(/[\{\}]/g, '')}_waiting_count`,
help: 'Waiting count',
async collect() {
const metric = await queue.getWaitingCount();

3
apps/worker/src/utils/session-handler.ts
View File

@@ -113,13 +113,12 @@ export async function getSessionEndJob(args: {
} | null> {
const state = await job.getState();
if (state !== 'delayed') {
logger.info(`[session-handler] Session end job is in "${state}" state`, {
logger.debug(`[session-handler] Session end job is in "${state}" state`, {
state,
retryCount,
jobTimestamp: new Date(job.timestamp).toISOString(),
jobDelta: Date.now() - job.timestamp,
jobId: job.id,
reqId: job.data.payload.properties?.__reqId ?? 'unknown',
payload: job.data.payload,
});
}