fix: overall perf improvements
* fix: ignore private ips * fix: performance related fixes * fix: simply event buffer * fix: default to 1 events queue shard * add: cleanup scripts * fix: comments * fix comments * fix * fix: groupmq * wip * fix: sync cachable * remove cluster names and add it behind env flag (if someone want to scale) * fix * wip * better logger * remove reqid and user agent * fix lock * remove wait_for_async_insert
This commit is contained in:
Carl-Gerhard Lindesvärd
committed by
GitHub
GitHub
parent
38cc53890a
commit
da59622dce
@@ -1,6 +1,9 @@
|
||||
import { type IServiceEvent, createEvent } from '@openpanel/db';
|
||||
import { eventBuffer } from '@openpanel/db';
|
||||
import { sessionsQueue } from '@openpanel/queue';
|
||||
import {
|
||||
type EventsQueuePayloadIncomingEvent,
|
||||
sessionsQueue,
|
||||
} from '@openpanel/queue';
|
||||
import type { Job } from 'bullmq';
|
||||
import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { incomingEvent } from './events.incoming-event';
|
||||
@@ -32,6 +35,28 @@ const geo = {
|
||||
latitude: 0,
|
||||
};
|
||||
|
||||
const uaInfo: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
|
||||
isServer: false,
|
||||
device: 'desktop',
|
||||
os: 'Windows',
|
||||
osVersion: '10',
|
||||
browser: 'Chrome',
|
||||
browserVersion: '91.0.4472.124',
|
||||
brand: '',
|
||||
model: '',
|
||||
};
|
||||
|
||||
const uaInfoServer: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
|
||||
isServer: true,
|
||||
device: 'server',
|
||||
os: '',
|
||||
osVersion: '',
|
||||
browser: '',
|
||||
browserVersion: '',
|
||||
brand: '',
|
||||
model: '',
|
||||
};
|
||||
|
||||
describe('incomingEvent', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
@@ -41,31 +66,29 @@ describe('incomingEvent', () => {
|
||||
const spySessionsQueueAdd = vi.spyOn(sessionsQueue, 'add');
|
||||
const timestamp = new Date();
|
||||
// Mock job data
|
||||
const jobData = {
|
||||
payload: {
|
||||
geo,
|
||||
event: {
|
||||
name: 'test_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
properties: { __path: 'https://example.com/test' },
|
||||
},
|
||||
headers: {
|
||||
'request-id': '123',
|
||||
'user-agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'openpanel-sdk-name': 'web',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
},
|
||||
projectId,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
|
||||
geo,
|
||||
event: {
|
||||
name: 'test_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
isTimestampFromThePast: false,
|
||||
properties: { __path: 'https://example.com/test' },
|
||||
},
|
||||
uaInfo,
|
||||
headers: {
|
||||
'request-id': '123',
|
||||
'user-agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'openpanel-sdk-name': 'web',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
},
|
||||
projectId,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
};
|
||||
|
||||
const job = { data: jobData } as Job;
|
||||
|
||||
// Execute the job
|
||||
await incomingEvent(job);
|
||||
await incomingEvent(jobData);
|
||||
|
||||
const event = {
|
||||
name: 'test_event',
|
||||
@@ -78,8 +101,6 @@ describe('incomingEvent', () => {
|
||||
properties: {
|
||||
__hash: undefined,
|
||||
__query: undefined,
|
||||
__user_agent: jobData.payload.headers['user-agent'],
|
||||
__reqId: jobData.payload.headers['request-id'],
|
||||
},
|
||||
createdAt: timestamp,
|
||||
country: 'US',
|
||||
@@ -92,16 +113,16 @@ describe('incomingEvent', () => {
|
||||
browser: 'Chrome',
|
||||
browserVersion: '91.0.4472.124',
|
||||
device: 'desktop',
|
||||
brand: undefined,
|
||||
model: undefined,
|
||||
brand: '',
|
||||
model: '',
|
||||
duration: 0,
|
||||
path: '/test',
|
||||
origin: 'https://example.com',
|
||||
referrer: '',
|
||||
referrerName: '',
|
||||
referrerType: '',
|
||||
sdkName: jobData.payload.headers['openpanel-sdk-name'],
|
||||
sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
|
||||
sdkName: jobData.headers['openpanel-sdk-name'],
|
||||
sdkVersion: jobData.headers['openpanel-sdk-version'],
|
||||
};
|
||||
|
||||
expect(spySessionsQueueAdd).toHaveBeenCalledWith(
|
||||
@@ -135,29 +156,27 @@ describe('incomingEvent', () => {
|
||||
|
||||
const timestamp = new Date();
|
||||
// Mock job data
|
||||
const jobData = {
|
||||
payload: {
|
||||
geo,
|
||||
event: {
|
||||
name: 'test_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
properties: { __path: 'https://example.com/test' },
|
||||
},
|
||||
headers: {
|
||||
'request-id': '123',
|
||||
'user-agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'openpanel-sdk-name': 'web',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
},
|
||||
projectId,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
|
||||
geo,
|
||||
event: {
|
||||
name: 'test_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
properties: { __path: 'https://example.com/test' },
|
||||
isTimestampFromThePast: false,
|
||||
},
|
||||
headers: {
|
||||
'request-id': '123',
|
||||
'user-agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'openpanel-sdk-name': 'web',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
},
|
||||
uaInfo,
|
||||
projectId,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
};
|
||||
|
||||
const job = { data: jobData } as Job;
|
||||
|
||||
const changeDelay = vi.fn();
|
||||
const updateData = vi.fn();
|
||||
spySessionsQueueGetJob.mockResolvedValueOnce({
|
||||
@@ -175,7 +194,7 @@ describe('incomingEvent', () => {
|
||||
},
|
||||
} as Partial<Job> as Job);
|
||||
// Execute the job
|
||||
await incomingEvent(job);
|
||||
await incomingEvent(jobData);
|
||||
|
||||
const event = {
|
||||
name: 'test_event',
|
||||
@@ -186,8 +205,6 @@ describe('incomingEvent', () => {
|
||||
properties: {
|
||||
__hash: undefined,
|
||||
__query: undefined,
|
||||
__user_agent: jobData.payload.headers['user-agent'],
|
||||
__reqId: jobData.payload.headers['request-id'],
|
||||
},
|
||||
createdAt: timestamp,
|
||||
country: 'US',
|
||||
@@ -200,16 +217,16 @@ describe('incomingEvent', () => {
|
||||
browser: 'Chrome',
|
||||
browserVersion: '91.0.4472.124',
|
||||
device: 'desktop',
|
||||
brand: undefined,
|
||||
model: undefined,
|
||||
brand: '',
|
||||
model: '',
|
||||
duration: 0,
|
||||
path: '/test',
|
||||
origin: 'https://example.com',
|
||||
referrer: '',
|
||||
referrerName: '',
|
||||
referrerType: '',
|
||||
sdkName: jobData.payload.headers['openpanel-sdk-name'],
|
||||
sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
|
||||
sdkName: jobData.headers['openpanel-sdk-name'],
|
||||
sdkVersion: jobData.headers['openpanel-sdk-version'],
|
||||
};
|
||||
|
||||
expect(spySessionsQueueAdd).toHaveBeenCalledTimes(0);
|
||||
@@ -220,29 +237,27 @@ describe('incomingEvent', () => {
|
||||
|
||||
it('should handle server events (with existing screen view)', async () => {
|
||||
const timestamp = new Date();
|
||||
const jobData = {
|
||||
payload: {
|
||||
geo,
|
||||
event: {
|
||||
name: 'server_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
properties: { custom_property: 'test_value' },
|
||||
profileId: 'profile-123',
|
||||
},
|
||||
headers: {
|
||||
'user-agent': 'OpenPanel Server/1.0',
|
||||
'openpanel-sdk-name': 'server',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
'request-id': '123',
|
||||
},
|
||||
projectId,
|
||||
currentDeviceId: '',
|
||||
previousDeviceId: '',
|
||||
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
|
||||
geo,
|
||||
event: {
|
||||
name: 'server_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
properties: { custom_property: 'test_value' },
|
||||
profileId: 'profile-123',
|
||||
isTimestampFromThePast: false,
|
||||
},
|
||||
headers: {
|
||||
'user-agent': 'OpenPanel Server/1.0',
|
||||
'openpanel-sdk-name': 'server',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
'request-id': '123',
|
||||
},
|
||||
projectId,
|
||||
currentDeviceId: '',
|
||||
previousDeviceId: '',
|
||||
uaInfo: uaInfoServer,
|
||||
};
|
||||
|
||||
const job = { data: jobData } as Job;
|
||||
|
||||
const mockLastScreenView = {
|
||||
deviceId: 'last-device-123',
|
||||
sessionId: 'last-session-456',
|
||||
@@ -268,7 +283,7 @@ describe('incomingEvent', () => {
|
||||
mockLastScreenView as IServiceEvent,
|
||||
);
|
||||
|
||||
await incomingEvent(job);
|
||||
await incomingEvent(jobData);
|
||||
|
||||
expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
|
||||
name: 'server_event',
|
||||
@@ -278,8 +293,6 @@ describe('incomingEvent', () => {
|
||||
projectId,
|
||||
properties: {
|
||||
custom_property: 'test_value',
|
||||
__user_agent: 'OpenPanel Server/1.0',
|
||||
__reqId: '123',
|
||||
__hash: undefined,
|
||||
__query: undefined,
|
||||
},
|
||||
@@ -311,33 +324,31 @@ describe('incomingEvent', () => {
|
||||
|
||||
it('should handle server events (without existing screen view)', async () => {
|
||||
const timestamp = new Date();
|
||||
const jobData = {
|
||||
payload: {
|
||||
geo,
|
||||
event: {
|
||||
name: 'server_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
properties: { custom_property: 'test_value' },
|
||||
profileId: 'profile-123',
|
||||
},
|
||||
headers: {
|
||||
'user-agent': 'OpenPanel Server/1.0',
|
||||
'openpanel-sdk-name': 'server',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
'request-id': '123',
|
||||
},
|
||||
projectId,
|
||||
currentDeviceId: '',
|
||||
previousDeviceId: '',
|
||||
const jobData: EventsQueuePayloadIncomingEvent['payload'] = {
|
||||
geo,
|
||||
event: {
|
||||
name: 'server_event',
|
||||
timestamp: timestamp.toISOString(),
|
||||
properties: { custom_property: 'test_value' },
|
||||
profileId: 'profile-123',
|
||||
isTimestampFromThePast: false,
|
||||
},
|
||||
headers: {
|
||||
'user-agent': 'OpenPanel Server/1.0',
|
||||
'openpanel-sdk-name': 'server',
|
||||
'openpanel-sdk-version': '1.0.0',
|
||||
'request-id': '123',
|
||||
},
|
||||
projectId,
|
||||
currentDeviceId: '',
|
||||
previousDeviceId: '',
|
||||
uaInfo: uaInfoServer,
|
||||
};
|
||||
|
||||
const job = { data: jobData } as Job;
|
||||
|
||||
// Mock getLastScreenView to return null
|
||||
vi.mocked(eventBuffer.getLastScreenView).mockResolvedValueOnce(null);
|
||||
|
||||
await incomingEvent(job);
|
||||
await incomingEvent(jobData);
|
||||
|
||||
expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
|
||||
name: 'server_event',
|
||||
@@ -347,8 +358,6 @@ describe('incomingEvent', () => {
|
||||
projectId,
|
||||
properties: {
|
||||
custom_property: 'test_value',
|
||||
__user_agent: 'OpenPanel Server/1.0',
|
||||
__reqId: '123',
|
||||
__hash: undefined,
|
||||
__query: undefined,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user