fix: overall perf improvements
* fix: ignore private ips * fix: performance related fixes * fix: simply event buffer * fix: default to 1 events queue shard * add: cleanup scripts * fix: comments * fix comments * fix * fix: groupmq * wip * fix: sync cachable * remove cluster names and add it behind env flag (if someone want to scale) * fix * wip * better logger * remove reqid and user agent * fix lock * remove wait_for_async_insert
This commit is contained in:
Carl-Gerhard Lindesvärd
committed by
GitHub
GitHub
parent
38cc53890a
commit
da59622dce
@@ -1,12 +1,11 @@
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { path, assocPath, pathOr, pick } from 'ramda';
|
||||
import { assocPath, pathOr, pick } from 'ramda';
|
||||
|
||||
import { checkDuplicatedEvent } from '@/utils/deduplicate';
|
||||
import { generateId } from '@openpanel/common';
|
||||
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
|
||||
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
|
||||
import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
|
||||
import { eventsGroupQueue } from '@openpanel/queue';
|
||||
import { getEventsGroupQueueShard } from '@openpanel/queue';
|
||||
import type {
|
||||
DecrementPayload,
|
||||
IdentifyPayload,
|
||||
@@ -37,10 +36,10 @@ export function getStringHeaders(headers: FastifyRequest['headers']) {
|
||||
}
|
||||
|
||||
function getIdentity(body: TrackHandlerPayload): IdentifyPayload | undefined {
|
||||
const identity = path<IdentifyPayload>(
|
||||
['properties', '__identify'],
|
||||
body.payload,
|
||||
);
|
||||
const identity =
|
||||
'properties' in body.payload
|
||||
? (body.payload?.properties?.__identify as IdentifyPayload | undefined)
|
||||
: undefined;
|
||||
|
||||
return (
|
||||
identity ||
|
||||
@@ -56,27 +55,28 @@ export function getTimestamp(
|
||||
timestamp: FastifyRequest['timestamp'],
|
||||
payload: TrackHandlerPayload['payload'],
|
||||
) {
|
||||
const safeTimestamp = new Date(timestamp || Date.now()).toISOString();
|
||||
const userDefinedTimestamp = path<string>(
|
||||
['properties', '__timestamp'],
|
||||
payload,
|
||||
);
|
||||
const safeTimestamp = timestamp || Date.now();
|
||||
const userDefinedTimestamp =
|
||||
'properties' in payload
|
||||
? (payload?.properties?.__timestamp as string | undefined)
|
||||
: undefined;
|
||||
|
||||
if (!userDefinedTimestamp) {
|
||||
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
|
||||
}
|
||||
|
||||
const clientTimestamp = new Date(userDefinedTimestamp);
|
||||
const clientTimestampNumber = clientTimestamp.getTime();
|
||||
|
||||
if (
|
||||
Number.isNaN(clientTimestamp.getTime()) ||
|
||||
clientTimestamp > new Date(safeTimestamp)
|
||||
Number.isNaN(clientTimestampNumber) ||
|
||||
clientTimestampNumber > safeTimestamp
|
||||
) {
|
||||
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
|
||||
}
|
||||
|
||||
return {
|
||||
timestamp: clientTimestamp.toISOString(),
|
||||
timestamp: clientTimestampNumber,
|
||||
isTimestampFromThePast: true,
|
||||
};
|
||||
}
|
||||
@@ -89,18 +89,19 @@ export async function handler(
|
||||
) {
|
||||
const timestamp = getTimestamp(request.timestamp, request.body.payload);
|
||||
const ip =
|
||||
path<string>(['properties', '__ip'], request.body.payload) ||
|
||||
request.clientIp;
|
||||
'properties' in request.body.payload &&
|
||||
request.body.payload.properties?.__ip
|
||||
? (request.body.payload.properties.__ip as string)
|
||||
: request.clientIp;
|
||||
const ua = request.headers['user-agent']!;
|
||||
const projectId = request.client?.projectId;
|
||||
|
||||
if (!projectId) {
|
||||
reply.status(400).send({
|
||||
return reply.status(400).send({
|
||||
status: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Missing projectId',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const identity = getIdentity(request.body);
|
||||
@@ -132,33 +133,7 @@ export async function handler(
|
||||
})
|
||||
: '';
|
||||
|
||||
if (
|
||||
await checkDuplicatedEvent({
|
||||
reply,
|
||||
payload: {
|
||||
...request.body,
|
||||
timestamp,
|
||||
previousDeviceId,
|
||||
currentDeviceId,
|
||||
},
|
||||
projectId,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const promises = [
|
||||
track({
|
||||
payload: request.body.payload,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
projectId,
|
||||
geo,
|
||||
headers: getStringHeaders(request.headers),
|
||||
timestamp: timestamp.timestamp,
|
||||
isTimestampFromThePast: timestamp.isTimestampFromThePast,
|
||||
}),
|
||||
];
|
||||
const promises = [];
|
||||
|
||||
// If we have more than one property in the identity object, we should identify the user
|
||||
// Otherwise its only a profileId and we should not identify the user
|
||||
@@ -173,23 +148,23 @@ export async function handler(
|
||||
);
|
||||
}
|
||||
|
||||
promises.push(
|
||||
track({
|
||||
payload: request.body.payload,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
projectId,
|
||||
geo,
|
||||
headers: getStringHeaders(request.headers),
|
||||
timestamp: timestamp.timestamp,
|
||||
isTimestampFromThePast: timestamp.isTimestampFromThePast,
|
||||
}),
|
||||
);
|
||||
|
||||
await Promise.all(promises);
|
||||
break;
|
||||
}
|
||||
case 'identify': {
|
||||
if (
|
||||
await checkDuplicatedEvent({
|
||||
reply,
|
||||
payload: {
|
||||
...request.body,
|
||||
timestamp,
|
||||
},
|
||||
projectId,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const geo = await getGeoLocation(ip);
|
||||
await identify({
|
||||
payload: request.body.payload,
|
||||
@@ -200,27 +175,13 @@ export async function handler(
|
||||
break;
|
||||
}
|
||||
case 'alias': {
|
||||
reply.status(400).send({
|
||||
return reply.status(400).send({
|
||||
status: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Alias is not supported',
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'increment': {
|
||||
if (
|
||||
await checkDuplicatedEvent({
|
||||
reply,
|
||||
payload: {
|
||||
...request.body,
|
||||
timestamp,
|
||||
},
|
||||
projectId,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
await increment({
|
||||
payload: request.body.payload,
|
||||
projectId,
|
||||
@@ -228,19 +189,6 @@ export async function handler(
|
||||
break;
|
||||
}
|
||||
case 'decrement': {
|
||||
if (
|
||||
await checkDuplicatedEvent({
|
||||
reply,
|
||||
payload: {
|
||||
...request.body,
|
||||
timestamp,
|
||||
},
|
||||
projectId,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
await decrement({
|
||||
payload: request.body.payload,
|
||||
projectId,
|
||||
@@ -248,12 +196,11 @@ export async function handler(
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
reply.status(400).send({
|
||||
return reply.status(400).send({
|
||||
status: 400,
|
||||
error: 'Bad Request',
|
||||
message: 'Invalid type',
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,7 +223,7 @@ async function track({
|
||||
projectId: string;
|
||||
geo: GeoLocation;
|
||||
headers: Record<string, string | undefined>;
|
||||
timestamp: string;
|
||||
timestamp: number;
|
||||
isTimestampFromThePast: boolean;
|
||||
}) {
|
||||
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
|
||||
@@ -285,8 +232,11 @@ async function track({
|
||||
? `${projectId}:${payload.profileId}`
|
||||
: `${projectId}:${generateId()}`
|
||||
: currentDeviceId;
|
||||
await eventsGroupQueue.add({
|
||||
orderMs: new Date(timestamp).getTime(),
|
||||
const jobId = [payload.name, timestamp, projectId, currentDeviceId, groupId]
|
||||
.filter(Boolean)
|
||||
.join('-');
|
||||
await getEventsGroupQueueShard(groupId).add({
|
||||
orderMs: timestamp,
|
||||
data: {
|
||||
projectId,
|
||||
headers,
|
||||
@@ -295,11 +245,13 @@ async function track({
|
||||
timestamp,
|
||||
isTimestampFromThePast,
|
||||
},
|
||||
uaInfo,
|
||||
geo,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
},
|
||||
groupId,
|
||||
jobId,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -322,8 +274,18 @@ async function identify({
|
||||
projectId,
|
||||
properties: {
|
||||
...(payload.properties ?? {}),
|
||||
...(geo ?? {}),
|
||||
...uaInfo,
|
||||
country: geo.country,
|
||||
city: geo.city,
|
||||
region: geo.region,
|
||||
longitude: geo.longitude,
|
||||
latitude: geo.latitude,
|
||||
os: uaInfo.os,
|
||||
os_version: uaInfo.osVersion,
|
||||
browser: uaInfo.browser,
|
||||
browser_version: uaInfo.browserVersion,
|
||||
device: uaInfo.device,
|
||||
brand: uaInfo.brand,
|
||||
model: uaInfo.model,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user