zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: performance related fixes

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-10-22 12:29:56 +02:00
parent 8bb0c87ec9
commit 1285ad85a2
60 changed files with 4264 additions and 2959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
apps/api/src/controllers/event.controller.ts
View File

@@ -2,10 +2,9 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { eventsGroupQueue } from '@openpanel/queue';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import type { PostEventPayload } from '@openpanel/sdk';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
import { generateId } from '@openpanel/common';
import { getGeoLocation } from '@openpanel/geo';
import { getStringHeaders, getTimestamp } from './track.controller';
@@ -44,28 +43,22 @@ export async function postEvent(
ua,
});
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
previousDeviceId,
currentDeviceId,
},
projectId,
})
) {
return;
}
const uaInfo = parseUserAgent(ua, request.body?.properties);
const groupId = uaInfo.isServer
? request.body?.profileId
? `${projectId}:${request.body?.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
const jobId = [
request.body.name,
timestamp,
projectId,
currentDeviceId,
groupId,
]
.filter(Boolean)
.join('-');
await getEventsGroupQueueShard(groupId).add({
orderMs: new Date(timestamp).getTime(),
data: {
projectId,
@@ -75,11 +68,13 @@ export async function postEvent(
timestamp,
isTimestampFromThePast,
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
},
groupId,
jobId,
});
reply.status(202).send('ok');

7
apps/api/src/controllers/live.controller.ts
View File

@@ -4,7 +4,7 @@ import superjson from 'superjson';
import type { WebSocket } from '@fastify/websocket';
import {
eventBuffer,
getProfileByIdCached,
getProfileById,
transformMinimalEvent,
} from '@openpanel/db';
import { setSuperJson } from '@openpanel/json';
@@ -92,10 +92,7 @@ export async function wsProjectEvents(
type,
async (event) => {
if (event.projectId === params.projectId) {
const profile = await getProfileByIdCached(
event.profileId,
event.projectId,
);
const profile = await getProfileById(event.profileId, event.projectId);
socket.send(
superjson.stringify(
access

8
apps/api/src/controllers/misc.controller.ts
View File

@@ -132,7 +132,7 @@ async function processImage(
): Promise<Buffer> {
// If it's an ICO file, just return it as-is (no conversion needed)
if (originalUrl && isIcoFile(originalUrl, contentType)) {
logger.info('Serving ICO file directly', {
logger.debug('Serving ICO file directly', {
originalUrl,
bufferSize: buffer.length,
});
@@ -140,7 +140,7 @@ async function processImage(
}
if (originalUrl && isSvgFile(originalUrl, contentType)) {
logger.info('Serving SVG file directly', {
logger.debug('Serving SVG file directly', {
originalUrl,
bufferSize: buffer.length,
});
@@ -149,7 +149,7 @@ async function processImage(
// If buffer isnt to big just return it as well
if (buffer.length < 5000) {
logger.info('Serving image directly without processing', {
logger.debug('Serving image directly without processing', {
originalUrl,
bufferSize: buffer.length,
});
@@ -193,7 +193,7 @@ async function processOgImage(
): Promise<Buffer> {
// If buffer is small enough, return it as-is
if (buffer.length < 10000) {
logger.info('Serving OG image directly without processing', {
logger.debug('Serving OG image directly without processing', {
originalUrl,
bufferSize: buffer.length,
});

63
apps/api/src/controllers/profile.controller.ts
View File

@@ -1,7 +1,6 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, pathOr } from 'ramda';
import { checkDuplicatedEvent, isDuplicatedEvent } from '@/utils/deduplicate';
import { parseUserAgent } from '@openpanel/common/server';
import { getProfileById, upsertProfile } from '@openpanel/db';
import { getGeoLocation } from '@openpanel/geo';
@@ -16,41 +15,39 @@ export async function updateProfile(
}>,
reply: FastifyReply,
) {
const { profileId, properties, ...rest } = request.body;
const payload = request.body;
const projectId = request.client!.projectId;
if (!projectId) {
return reply.status(400).send('No projectId');
}
const ip = request.clientIp;
const ua = request.headers['user-agent']!;
const uaInfo = parseUserAgent(ua, properties);
const uaInfo = parseUserAgent(ua, payload.properties);
const geo = await getGeoLocation(ip);
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
},
projectId,
})
) {
return;
}
await upsertProfile({
id: profileId,
...payload,
id: payload.profileId,
isExternal: true,
projectId,
properties: {
...(properties ?? {}),
...(ip ? geo : {}),
...uaInfo,
...(payload.properties ?? {}),
country: geo.country,
city: geo.city,
region: geo.region,
longitude: geo.longitude,
latitude: geo.latitude,
os: uaInfo.os,
os_version: uaInfo.osVersion,
browser: uaInfo.browser,
browser_version: uaInfo.browserVersion,
device: uaInfo.device,
brand: uaInfo.brand,
model: uaInfo.model,
},
...rest,
});
reply.status(202).send(profileId);
reply.status(202).send(payload.profileId);
}
export async function incrementProfileProperty(
@@ -65,18 +62,6 @@ export async function incrementProfileProperty(
return reply.status(400).send('No projectId');
}
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
},
projectId,
})
) {
return;
}
const profile = await getProfileById(profileId, projectId);
if (!profile) {
return reply.status(404).send('Not found');
@@ -119,18 +104,6 @@ export async function decrementProfileProperty(
return reply.status(400).send('No projectId');
}
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
},
projectId,
})
) {
return;
}
const profile = await getProfileById(profileId, projectId);
if (!profile) {
return reply.status(404).send('Not found');

175
apps/api/src/controllers/track.controller.ts
View File

@@ -1,12 +1,14 @@
import type { FastifyReply, FastifyRequest } from 'fastify';
import { path, assocPath, pathOr, pick } from 'ramda';
import { assocPath, pathOr, pick } from 'ramda';
import { checkDuplicatedEvent } from '@/utils/deduplicate';
import { logger } from '@/utils/logger';
import { generateId } from '@openpanel/common';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
import { eventsGroupQueue } from '@openpanel/queue';
import type { ILogger } from '@openpanel/logger';
import { getEventsGroupQueueShard } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import type {
DecrementPayload,
IdentifyPayload,
@@ -37,10 +39,10 @@ export function getStringHeaders(headers: FastifyRequest['headers']) {
}
function getIdentity(body: TrackHandlerPayload): IdentifyPayload | undefined {
const identity = path<IdentifyPayload>(
['properties', '__identify'],
body.payload,
);
const identity =
'properties' in body.payload
? (body.payload?.properties?.__identify as IdentifyPayload | undefined)
: undefined;
return (
identity ||
@@ -56,27 +58,28 @@ export function getTimestamp(
timestamp: FastifyRequest['timestamp'],
payload: TrackHandlerPayload['payload'],
) {
const safeTimestamp = new Date(timestamp || Date.now()).toISOString();
const userDefinedTimestamp = path<string>(
['properties', '__timestamp'],
payload,
);
const safeTimestamp = timestamp || Date.now();
const userDefinedTimestamp =
'properties' in payload
? (payload?.properties?.__timestamp as string | undefined)
: undefined;
if (!userDefinedTimestamp) {
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
}
const clientTimestamp = new Date(userDefinedTimestamp);
const clientTimestampNumber = clientTimestamp.getTime();
if (
Number.isNaN(clientTimestamp.getTime()) ||
clientTimestamp > new Date(safeTimestamp)
Number.isNaN(clientTimestampNumber) ||
clientTimestampNumber > safeTimestamp
) {
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
}
return {
timestamp: clientTimestamp.toISOString(),
timestamp: clientTimestampNumber,
isTimestampFromThePast: true,
};
}
@@ -89,18 +92,19 @@ export async function handler(
) {
const timestamp = getTimestamp(request.timestamp, request.body.payload);
const ip =
path<string>(['properties', '__ip'], request.body.payload) ||
request.clientIp;
'properties' in request.body.payload &&
request.body.payload.properties?.__ip
? (request.body.payload.properties.__ip as string)
: request.clientIp;
const ua = request.headers['user-agent']!;
const projectId = request.client?.projectId;
if (!projectId) {
reply.status(400).send({
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Missing projectId',
});
return;
}
const identity = getIdentity(request.body);
@@ -132,33 +136,7 @@ export async function handler(
})
: '';
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
previousDeviceId,
currentDeviceId,
},
projectId,
})
) {
return;
}
const promises = [
track({
payload: request.body.payload,
currentDeviceId,
previousDeviceId,
projectId,
geo,
headers: getStringHeaders(request.headers),
timestamp: timestamp.timestamp,
isTimestampFromThePast: timestamp.isTimestampFromThePast,
}),
];
const promises = [];
// If we have more than one property in the identity object, we should identify the user
// Otherwise its only a profileId and we should not identify the user
@@ -173,23 +151,24 @@ export async function handler(
);
}
promises.push(
track({
log: request.log.info,
payload: request.body.payload,
currentDeviceId,
previousDeviceId,
projectId,
geo,
headers: getStringHeaders(request.headers),
timestamp: timestamp.timestamp,
isTimestampFromThePast: timestamp.isTimestampFromThePast,
}),
);
await Promise.all(promises);
break;
}
case 'identify': {
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
},
projectId,
})
) {
return;
}
const geo = await getGeoLocation(ip);
await identify({
payload: request.body.payload,
@@ -200,27 +179,13 @@ export async function handler(
break;
}
case 'alias': {
reply.status(400).send({
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Alias is not supported',
});
break;
}
case 'increment': {
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
},
projectId,
})
) {
return;
}
await increment({
payload: request.body.payload,
projectId,
@@ -228,19 +193,6 @@ export async function handler(
break;
}
case 'decrement': {
if (
await checkDuplicatedEvent({
reply,
payload: {
...request.body,
timestamp,
},
projectId,
})
) {
return;
}
await decrement({
payload: request.body.payload,
projectId,
@@ -248,12 +200,11 @@ export async function handler(
break;
}
default: {
reply.status(400).send({
return reply.status(400).send({
status: 400,
error: 'Bad Request',
message: 'Invalid type',
});
break;
}
}
@@ -269,6 +220,7 @@ async function track({
headers,
timestamp,
isTimestampFromThePast,
log,
}: {
payload: TrackPayload;
currentDeviceId: string;
@@ -276,8 +228,9 @@ async function track({
projectId: string;
geo: GeoLocation;
headers: Record<string, string | undefined>;
timestamp: string;
timestamp: number;
isTimestampFromThePast: boolean;
log: any;
}) {
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
const groupId = uaInfo.isServer
@@ -285,8 +238,14 @@ async function track({
? `${projectId}:${payload.profileId}`
: `${projectId}:${generateId()}`
: currentDeviceId;
await eventsGroupQueue.add({
orderMs: new Date(timestamp).getTime(),
const jobId = [payload.name, timestamp, projectId, currentDeviceId, groupId]
.filter(Boolean)
.join('-');
await getRedisCache().incr('track:counter');
log('track handler', {
jobId: jobId,
groupId: groupId,
timestamp: timestamp,
data: {
projectId,
headers,
@@ -295,11 +254,29 @@ async function track({
timestamp,
isTimestampFromThePast,
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
},
});
await getEventsGroupQueueShard(groupId).add({
orderMs: timestamp,
data: {
projectId,
headers,
event: {
...payload,
timestamp,
isTimestampFromThePast,
},
uaInfo,
geo,
currentDeviceId,
previousDeviceId,
},
groupId,
jobId,
});
}
@@ -322,8 +299,18 @@ async function identify({
projectId,
properties: {
...(payload.properties ?? {}),
...(geo ?? {}),
...uaInfo,
country: geo.country,
city: geo.city,
region: geo.region,
longitude: geo.longitude,
latitude: geo.latitude,
os: uaInfo.os,
os_version: uaInfo.osVersion,
browser: uaInfo.browser,
browser_version: uaInfo.browserVersion,
device: uaInfo.device,
brand: uaInfo.brand,
model: uaInfo.model,
},
});
}