zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: overall perf improvements

Browse Source 
* fix: ignore private ips

* fix: performance related fixes

* fix: simply event buffer

* fix: default to 1 events queue shard

* add: cleanup scripts

* fix: comments

* fix comments

* fix

* fix: groupmq

* wip

* fix: sync cachable

* remove cluster names and add it behind env flag (if someone want to scale)

* fix

* wip

* better logger

* remove reqid and user agent

* fix lock

* remove wait_for_async_insert
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-15 22:13:59 +01:00
committed by GitHub
parent 38cc53890a
commit da59622dce
66 changed files with 5042 additions and 3860 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3457
apps/api/src/bots/bots.ts
View File

File diff suppressed because it is too large Load Diff

58
apps/api/src/bots/index.ts
View File

@@ -1,19 +1,47 @@
import { cacheable, cacheableLru } from '@openpanel/redis';
import bots from './bots';
export function isBot(ua: string) {
const res = bots.find((bot) => {
if (new RegExp(bot.regex).test(ua)) {
return true;
}
return false;
});
if (!res) {
return null;
// Pre-compile regex patterns at module load time
const compiledBots = bots.map((bot) => {
if ('regex' in bot) {
return {
...bot,
compiledRegex: new RegExp(bot.regex),
};
}
return bot;
});
return {
name: res.name,
type: 'category' in res ? res.category : 'Unknown',
};
}
const regexBots = compiledBots.filter((bot) => 'compiledRegex' in bot);
const includesBots = compiledBots.filter((bot) => 'includes' in bot);
export const isBot = cacheableLru(
'is-bot',
(ua: string) => {
// Check simple string patterns first (fast)
for (const bot of includesBots) {
if (ua.includes(bot.includes)) {
return {
name: bot.name,
type: 'category' in bot ? bot.category : 'Unknown',
};
}
}
// Check regex patterns (slower)
for (const bot of regexBots) {
if (bot.compiledRegex.test(ua)) {
return {
name: bot.name,
type: 'category' in bot ? bot.category : 'Unknown',
};
}
}
return null;
},
{
maxSize: 1000,
ttl: 60 * 5,
},
);

31
apps/api/src/controllers/event.controller.ts
View File

@@ -2,10 +2,9 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { eventsGroupQueue } from '@openpanel/queue';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import type { PostEventPayload } from '@openpanel/sdk';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
import { generateId } from '@openpanel/common';
import { getGeoLocation } from '@openpanel/geo';
import { getStringHeaders, getTimestamp } from './track.controller';
@@ -44,28 +43,22 @@ export async function postEvent(
ua,
});
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
previousDeviceId,
currentDeviceId,
},
projectId,
})
) {
return;
}
const uaInfo = parseUserAgent(ua, request.body?.properties);
const groupId = uaInfo.isServer
? request.body?.profileId
? `${projectId}:${request.body?.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
const jobId = [
request.body.name,
timestamp,
projectId,
currentDeviceId,
groupId,
]
.filter(Boolean)
.join('-');
await getEventsGroupQueueShard(groupId).add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
@@ -75,11 +68,13 @@ export async function postEvent(
timestamp,
isTimestampFromThePast,
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
},
groupId,
jobId,
});
reply.status(202).send('ok');

7
apps/api/src/controllers/live.controller.ts
View File

@@ -4,7 +4,7 @@ import superjson from 'superjson';
import type { WebSocket } from '@fastify/websocket';
import {
eventBuffer,
getProfileByIdCached,
getProfileById,
transformMinimalEvent,
} from '@openpanel/db';
import { setSuperJson } from '@openpanel/json';
@@ -92,10 +92,7 @@ export async function wsProjectEvents(
type,
async (event) => {
if (event.projectId === params.projectId) {
const profile = await getProfileByIdCached(
event.profileId,
event.projectId,
);
const profile = await getProfileById(event.profileId, event.projectId);
socket.send(
superjson.stringify(
access

8
apps/api/src/controllers/misc.controller.ts
View File

@@ -132,7 +132,7 @@ async function processImage(
): Promise<Buffer> {
// If it's an ICO file, just return it as-is (no conversion needed)
if (originalUrl && isIcoFile(originalUrl, contentType)) {
logger.info('Serving ICO file directly', {
logger.debug('Serving ICO file directly', {
originalUrl,
bufferSize: buffer.length,
});
@@ -140,7 +140,7 @@ async function processImage(
}
if (originalUrl && isSvgFile(originalUrl, contentType)) {
logger.info('Serving SVG file directly', {
logger.debug('Serving SVG file directly', {
originalUrl,
bufferSize: buffer.length,
});
@@ -149,7 +149,7 @@ async function processImage(
// If buffer isnt to big just return it as well
if (buffer.length < 5000) {
logger.info('Serving image directly without processing', {
logger.debug('Serving image directly without processing', {
originalUrl,
bufferSize: buffer.length,
});
@@ -193,7 +193,7 @@ async function processOgImage(
): Promise<Buffer> {
// If buffer is small enough, return it as-is
if (buffer.length < 10000) {
logger.info('Serving OG image directly without processing', {
logger.debug('Serving OG image directly without processing', {
originalUrl,
bufferSize: buffer.length,
});

63
apps/api/src/controllers/profile.controller.ts
View File

@@ -1,7 +1,6 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, pathOr } from 'ramda';
import { checkDuplicatedEvent, isDuplicatedEvent } from '@/utils/deduplicate';
import { parseUserAgent } from '@openpanel/common/server';
import { getProfileById, upsertProfile } from '@openpanel/db';
import { getGeoLocation } from '@openpanel/geo';
@@ -16,41 +15,39 @@ export async function updateProfile(
}>,
reply: FastifyReply,
) {
const { profileId, properties, ...rest } = request.body;
const payload = request.body;
const projectId = request.client!.projectId;
if (!projectId) {
return reply.status(400).send('No projectId');
}
const ip = request.clientIp;
const ua = request.headers['user-agent']!;
const uaInfo = parseUserAgent(ua, properties);
const uaInfo = parseUserAgent(ua, payload.properties);
const geo = await getGeoLocation(ip);
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
},
projectId,
})
) {
return;
}
await upsertProfile({
id: profileId,
...payload,
id: payload.profileId,
isExternal: true,
projectId,
properties: {
...(properties ?? {}),
...(ip ? geo : {}),
...uaInfo,
...(payload.properties ?? {}),
country: geo.country,
city: geo.city,
region: geo.region,
longitude: geo.longitude,
latitude: geo.latitude,
os: uaInfo.os,
os_version: uaInfo.osVersion,
browser: uaInfo.browser,
browser_version: uaInfo.browserVersion,
device: uaInfo.device,
brand: uaInfo.brand,
model: uaInfo.model,
},
...rest,
});
reply.status(202).send(profileId);
reply.status(202).send(payload.profileId);
}
export async function incrementProfileProperty(
@@ -65,18 +62,6 @@ export async function incrementProfileProperty(
return reply.status(400).send('No projectId');
}
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
},
projectId,
})
) {
return;
}
const profile = await getProfileById(profileId, projectId);
if (!profile) {
return reply.status(404).send('Not found');
@@ -119,18 +104,6 @@ export async function decrementProfileProperty(
return reply.status(400).send('No projectId');
}
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
},
projectId,
})
) {
return;
}
const profile = await getProfileById(profileId, projectId);
if (!profile) {
return reply.status(404).send('Not found');

150
apps/api/src/controllers/track.controller.ts
View File

@@ -1,12 +1,11 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { path, assocPath, pathOr, pick } from 'ramda';
import { assocPath, pathOr, pick } from 'ramda';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
import { generateId } from '@openpanel/common';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { eventsGroupQueue } from '@openpanel/queue';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import type {
DecrementPayload,
IdentifyPayload,
@@ -37,10 +36,10 @@ export function getStringHeaders(headers: FastifyRequest['headers']) {
}
function getIdentity(body: TrackHandlerPayload): IdentifyPayload | undefined {
const identity = path<IdentifyPayload>(
['properties', '__identify'],
body.payload,
);
const identity =
'properties' in body.payload
? (body.payload?.properties?.__identify as IdentifyPayload | undefined)
: undefined;
return (
identity ||
@@ -56,27 +55,28 @@ export function getTimestamp(
timestamp: FastifyRequest['timestamp'],
payload: TrackHandlerPayload['payload'],
) {
const safeTimestamp = new Date(timestamp || Date.now()).toISOString();
const userDefinedTimestamp = path<string>(
['properties', '__timestamp'],
payload,
);
const safeTimestamp = timestamp || Date.now();
const userDefinedTimestamp =
'properties' in payload
? (payload?.properties?.__timestamp as string | undefined)
: undefined;
if (!userDefinedTimestamp) {
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
}
const clientTimestamp = new Date(userDefinedTimestamp);
const clientTimestampNumber = clientTimestamp.getTime();
if (
Number.isNaN(clientTimestamp.getTime()) ||
clientTimestamp > new Date(safeTimestamp)
Number.isNaN(clientTimestampNumber) ||
clientTimestampNumber > safeTimestamp
) {
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
}
return {
timestamp: clientTimestamp.toISOString(),
timestamp: clientTimestampNumber,
isTimestampFromThePast: true,
};
}
@@ -89,18 +89,19 @@ export async function handler(
) {
const timestamp = getTimestamp(request.timestamp, request.body.payload);
const ip =
path<string>(['properties', '__ip'], request.body.payload) ||
request.clientIp;
'properties' in request.body.payload &&
request.body.payload.properties?.__ip
? (request.body.payload.properties.__ip as string)
: request.clientIp;
const ua = request.headers['user-agent']!;
const projectId = request.client?.projectId;
if (!projectId) {
reply.status(400).send({
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Missing projectId',
});
return;
}
const identity = getIdentity(request.body);
@@ -132,33 +133,7 @@ export async function handler(
})
: '';
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
previousDeviceId,
currentDeviceId,
},
projectId,
})
) {
return;
}
const promises = [
track({
payload: request.body.payload,
currentDeviceId,
previousDeviceId,
projectId,
geo,
headers: getStringHeaders(request.headers),
timestamp: timestamp.timestamp,
isTimestampFromThePast: timestamp.isTimestampFromThePast,
}),
];
const promises = [];
// If we have more than one property in the identity object, we should identify the user
// Otherwise its only a profileId and we should not identify the user
@@ -173,23 +148,23 @@ export async function handler(
);
}
promises.push(
track({
payload: request.body.payload,
currentDeviceId,
previousDeviceId,
projectId,
geo,
headers: getStringHeaders(request.headers),
timestamp: timestamp.timestamp,
isTimestampFromThePast: timestamp.isTimestampFromThePast,
}),
);
await Promise.all(promises);
break;
}
case 'identify': {
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
},
projectId,
})
) {
return;
}
const geo = await getGeoLocation(ip);
await identify({
payload: request.body.payload,
@@ -200,27 +175,13 @@ export async function handler(
break;
}
case 'alias': {
reply.status(400).send({
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Alias is not supported',
});
break;
}
case 'increment': {
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
},
projectId,
})
) {
return;
}
await increment({
payload: request.body.payload,
projectId,
@@ -228,19 +189,6 @@ export async function handler(
break;
}
case 'decrement': {
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
},
projectId,
})
) {
return;
}
await decrement({
payload: request.body.payload,
projectId,
@@ -248,12 +196,11 @@ export async function handler(
break;
}
default: {
reply.status(400).send({
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Invalid type',
});
break;
}
}
@@ -276,7 +223,7 @@ async function track({
projectId: string;
geo: GeoLocation;
headers: Record<string, string | undefined>;
timestamp: string;
timestamp: number;
isTimestampFromThePast: boolean;
}) {
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
@@ -285,8 +232,11 @@ async function track({
? `${projectId}:${payload.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
const jobId = [payload.name, timestamp, projectId, currentDeviceId, groupId]
.filter(Boolean)
.join('-');
await getEventsGroupQueueShard(groupId).add({
orderMs: timestamp,
data: {
projectId,
headers,
@@ -295,11 +245,13 @@ async function track({
timestamp,
isTimestampFromThePast,
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
},
groupId,
jobId,
});
}
@@ -322,8 +274,18 @@ async function identify({
projectId,
properties: {
...(payload.properties ?? {}),
...(geo ?? {}),
...uaInfo,
country: geo.country,
city: geo.city,
region: geo.region,
longitude: geo.longitude,
latitude: geo.latitude,
os: uaInfo.os,
os_version: uaInfo.osVersion,
browser: uaInfo.browser,
browser_version: uaInfo.browserVersion,
device: uaInfo.device,
brand: uaInfo.brand,
model: uaInfo.model,
},
});
}

28
apps/api/src/hooks/duplicate.hook.ts Normal file
View File

@@ -0,0 +1,28 @@
import { isDuplicatedEvent } from '@/utils/deduplicate';
import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
import type { FastifyReply, FastifyRequest } from 'fastify';
export async function duplicateHook(
req: FastifyRequest<{
Body: PostEventPayload | TrackHandlerPayload;
}>,
reply: FastifyReply,
) {
const ip = req.clientIp;
const origin = req.headers.origin;
const clientId = req.headers['openpanel-client-id'];
const shouldCheck = ip && origin && clientId;
const isDuplicate = shouldCheck
? await isDuplicatedEvent({
ip,
origin,
payload: req.body,
projectId: clientId as string,
})
: false;
if (isDuplicate) {
return reply.status(200).send('Duplicate event');
}
}

16
apps/api/src/hooks/fix.hook.ts
View File

@@ -1,16 +0,0 @@
import type { FastifyRequest } from 'fastify';
export async function fixHook(request: FastifyRequest) {
const ua = request.headers['user-agent'];
// Swift SDK issue: https://github.com/Openpanel-dev/swift-sdk/commit/d588fa761a36a33f3b78eb79d83bfd524e3c7144
if (ua) {
const regex = /OpenPanel\/(\d+\.\d+\.\d+)\sOpenPanel\/(\d+\.\d+\.\d+)/;
const match = ua.match(regex);
if (match) {
request.headers['user-agent'] = ua.replace(
regex,
`OpenPanel/${match[1]}`,
);
}
}
}

2
apps/api/src/index.ts
View File

@@ -28,7 +28,6 @@ import {
liveness,
readiness,
} from './controllers/healthcheck.controller';
import { fixHook } from './hooks/fix.hook';
import { ipHook } from './hooks/ip.hook';
import { requestIdHook } from './hooks/request-id.hook';
import { requestLoggingHook } from './hooks/request-logging.hook';
@@ -125,7 +124,6 @@ const startServer = async () => {
fastify.addHook('onRequest', requestIdHook);
fastify.addHook('onRequest', timestampHook);
fastify.addHook('onRequest', ipHook);
fastify.addHook('onRequest', fixHook);
fastify.addHook('onResponse', requestLoggingHook);
fastify.register(compress, {

2
apps/api/src/routes/event.router.ts
View File

@@ -2,9 +2,11 @@ import * as controller from '@/controllers/event.controller';
import type { FastifyPluginCallback } from 'fastify';
import { clientHook } from '@/hooks/client.hook';
import { duplicateHook } from '@/hooks/duplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
const eventRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preValidation', duplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.addHook('preHandler', isBotHook);

2
apps/api/src/routes/track.router.ts
View File

@@ -2,9 +2,11 @@ import { handler } from '@/controllers/track.controller';
import type { FastifyPluginCallback } from 'fastify';
import { clientHook } from '@/hooks/client.hook';
import { duplicateHook } from '@/hooks/duplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
const trackRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preValidation', duplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.addHook('preHandler', isBotHook);

9
apps/api/src/utils/auth.ts
View File

@@ -3,6 +3,7 @@ import type { FastifyRequest, RawRequestDefaultExpression } from 'fastify';
import { verifyPassword } from '@openpanel/common/server';
import type { IServiceClientWithProject } from '@openpanel/db';
import { ClientType, getClientByIdCached } from '@openpanel/db';
import { getCache } from '@openpanel/redis';
import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
import type {
IProjectFilterIp,
@@ -135,7 +136,13 @@ export async function validateSdkRequest(
}
if (client.secret && clientSecret) {
if (await verifyPassword(clientSecret, client.secret)) {
const isVerified = await getCache(
`client:auth:${clientId}:${Buffer.from(clientSecret).toString('base64')}`,
60 * 5,
async () => await verifyPassword(clientSecret, client.secret!),
true,
);
if (isVerified) {
return client;
}
}

28
apps/api/src/utils/deduplicate.ts
View File

@@ -1,11 +1,14 @@
import { getLock } from '@openpanel/redis';
import fastJsonStableHash from 'fast-json-stable-hash';
import type { FastifyReply } from 'fastify';
export async function isDuplicatedEvent({
ip,
origin,
payload,
projectId,
}: {
ip: string;
origin: string;
payload: Record<string, any>;
projectId: string;
}) {
@@ -13,6 +16,8 @@ export async function isDuplicatedEvent({
`fastify:deduplicate:${fastJsonStableHash.hash(
{
...payload,
ip,
origin,
projectId,
},
'md5',
@@ -27,24 +32,3 @@ export async function isDuplicatedEvent({
return true;
}
export async function checkDuplicatedEvent({
reply,
payload,
projectId,
}: {
reply: FastifyReply;
payload: Record<string, any>;
projectId: string;
}) {
if (await isDuplicatedEvent({ payload, projectId })) {
reply.log.info('duplicated event', {
payload,
projectId,
});
reply.status(200).send('duplicated');
return true;
}
return false;
}

4
apps/api/src/utils/graceful-shutdown.ts
View File

@@ -1,7 +1,7 @@
import { ch, db } from '@openpanel/db';
import {
cronQueue,
eventsGroupQueue,
eventsGroupQueues,
miscQueue,
notificationQueue,
sessionsQueue,
@@ -71,7 +71,7 @@ export async function shutdown(
// Step 6: Close Bull queues (graceful shutdown of queue state)
try {
await Promise.all([
eventsGroupQueue.close(),
...eventsGroupQueues.map((queue) => queue.close()),
sessionsQueue.close(),
cronQueue.close(),
miscQueue.close(),