zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

a looooot

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-02-22 21:50:30 +01:00
parent 1d800835b8
commit 9c92803c4c
61 changed files with 2689 additions and 681 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/sdk-api/src/bots/index.ts
View File

@@ -14,6 +14,6 @@ export function isBot(ua: string) {
return {
name: res.name,
type: res.category,
type: res.category || 'Unknown',
};
}

100
apps/sdk-api/src/controllers/event.controller.ts
View File

@@ -1,12 +1,13 @@
import { isBot } from '@/bots';
import { getClientIp, parseIp } from '@/utils/parseIp';
import { getReferrerWithQuery, parseReferrer } from '@/utils/parseReferrer';
import { parseUserAgent } from '@/utils/parseUserAgent';
import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { omit } from 'ramda';
import { generateProfileId, getTime, toISOString } from '@mixan/common';
import { generateDeviceId, getTime, toISOString } from '@mixan/common';
import type { IServiceCreateEventPayload } from '@mixan/db';
import { getSalts } from '@mixan/db';
import { createBotEvent, getEvents, getSalts } from '@mixan/db';
import type { JobsOptions } from '@mixan/queue';
import { eventsQueue, findJobByPrefix } from '@mixan/queue';
import type { PostEventPayload } from '@mixan/types';
@@ -66,9 +67,10 @@ export async function postEvent(
}>,
reply: FastifyReply
) {
let profileId: string | null = null;
let deviceId: string | null = null;
const projectId = request.projectId;
const body = request.body;
const profileId = body.profileId ?? '';
const createdAt = new Date(body.timestamp);
const url = body.properties?.path;
const { path, hash, query } = parsePath(url);
@@ -81,68 +83,118 @@ export async function postEvent(
const ua = request.headers['user-agent']!;
const uaInfo = parseUserAgent(ua);
const salts = await getSalts();
const currentProfileId = generateProfileId({
const currentDeviceId = generateDeviceId({
salt: salts.current,
origin,
ip,
ua,
});
const previousProfileId = generateProfileId({
const previousProfileId = generateDeviceId({
salt: salts.previous,
origin,
ip,
ua,
});
const isServerEvent = !ip && !origin && !isUserAgentSet(ua);
if (isServerEvent) {
const [event] = await getEvents(
`SELECT * FROM events WHERE name = 'screen_view' AND profile_id = '${profileId}' AND project_id = '${projectId}' ORDER BY created_at DESC LIMIT 1`
);
eventsQueue.add('event', {
type: 'createEvent',
payload: {
name: body.name,
deviceId: event?.deviceId || '',
profileId,
projectId,
properties: body.properties ?? {},
createdAt,
country: event?.country ?? '',
city: event?.city ?? '',
region: event?.region ?? '',
continent: event?.continent ?? '',
os: event?.os ?? '',
osVersion: event?.osVersion ?? '',
browser: event?.browser ?? '',
browserVersion: event?.browserVersion ?? '',
device: event?.device ?? '',
brand: event?.brand ?? '',
model: event?.model ?? '',
duration: 0,
path: event?.path ?? '',
referrer: event?.referrer ?? '',
referrerName: event?.referrerName ?? '',
referrerType: event?.referrerType ?? '',
profile: undefined,
meta: undefined,
},
});
return reply.status(200).send('');
}
const bot = isBot(ua);
if (bot) {
await createBotEvent({
...bot,
projectId,
createdAt: new Date(body.timestamp),
});
return reply.status(200).send('');
}
const [geo, eventsJobs] = await Promise.all([
parseIp(ip),
eventsQueue.getJobs(['delayed']),
]);
// find session_end job
const sessionEndJobCurrentProfileId = findJobByPrefix(
const sessionEndJobCurrentDeviceId = findJobByPrefix(
eventsJobs,
`sessionEnd:${projectId}:${currentProfileId}:`
`sessionEnd:${projectId}:${currentDeviceId}:`
);
const sessionEndJobPreviousProfileId = findJobByPrefix(
const sessionEndJobPreviousDeviceId = findJobByPrefix(
eventsJobs,
`sessionEnd:${projectId}:${previousProfileId}:`
);
const createSessionStart =
!sessionEndJobCurrentProfileId && !sessionEndJobPreviousProfileId;
!sessionEndJobCurrentDeviceId && !sessionEndJobPreviousDeviceId;
if (sessionEndJobCurrentProfileId && !sessionEndJobPreviousProfileId) {
if (sessionEndJobCurrentDeviceId && !sessionEndJobPreviousDeviceId) {
console.log('found session current');
profileId = currentProfileId;
const diff = Date.now() - sessionEndJobCurrentProfileId.timestamp;
sessionEndJobCurrentProfileId.changeDelay(diff + SESSION_END_TIMEOUT);
} else if (!sessionEndJobCurrentProfileId && sessionEndJobPreviousProfileId) {
deviceId = currentDeviceId;
const diff = Date.now() - sessionEndJobCurrentDeviceId.timestamp;
sessionEndJobCurrentDeviceId.changeDelay(diff + SESSION_END_TIMEOUT);
} else if (!sessionEndJobCurrentDeviceId && sessionEndJobPreviousDeviceId) {
console.log('found session previous');
profileId = previousProfileId;
const diff = Date.now() - sessionEndJobPreviousProfileId.timestamp;
sessionEndJobPreviousProfileId.changeDelay(diff + SESSION_END_TIMEOUT);
deviceId = previousProfileId;
const diff = Date.now() - sessionEndJobPreviousDeviceId.timestamp;
sessionEndJobPreviousDeviceId.changeDelay(diff + SESSION_END_TIMEOUT);
} else {
console.log('new session with current');
profileId = currentProfileId;
deviceId = currentDeviceId;
// Queue session end
eventsQueue.add(
'event',
{
type: 'createSessionEnd',
payload: {
profileId,
deviceId,
},
},
{
delay: SESSION_END_TIMEOUT,
jobId: `sessionEnd:${projectId}:${profileId}:${Date.now()}`,
jobId: `sessionEnd:${projectId}:${deviceId}:${Date.now()}`,
}
);
}
const payload: Omit<IServiceCreateEventPayload, 'id'> = {
name: body.name,
deviceId,
profileId,
projectId,
properties: Object.assign({}, omit(['path', 'referrer'], body.properties), {
@@ -170,7 +222,7 @@ export async function postEvent(
meta: undefined,
};
const job = findJobByPrefix(eventsJobs, `event:${projectId}:${profileId}:`);
const job = findJobByPrefix(eventsJobs, `event:${projectId}:${deviceId}:`);
if (job?.isDelayed && job.data.type === 'createEvent') {
const prevEvent = job.data.payload;
@@ -208,7 +260,7 @@ export async function postEvent(
const options: JobsOptions = {};
if (payload.name === 'screen_view') {
options.delay = SESSION_TIMEOUT;
options.jobId = `event:${projectId}:${profileId}:${Date.now()}`;
options.jobId = `event:${projectId}:${deviceId}:${Date.now()}`;
}
// Queue current event
@@ -221,5 +273,5 @@ export async function postEvent(
options
);
reply.status(202).send(profileId);
reply.status(202).send(deviceId);
}

2
apps/sdk-api/src/controllers/misc.controller.ts
View File

@@ -85,8 +85,6 @@ export async function getFavicon(
// TRY FAVICON.ICO
const buffer = await getImageBuffer(`${origin}/favicon.ico`);
console.log('buffer', buffer?.length);
if (buffer && buffer.byteLength > 0) {
return sendBuffer(buffer, hostname);
}

253
apps/sdk-api/src/controllers/profile.controller.ts
View File

@@ -1,195 +1,38 @@
import { getClientIp, parseIp } from '@/utils/parseIp';
import { parseUserAgent } from '@/utils/parseUserAgent';
import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, mergeDeepRight, path } from 'ramda';
import { assocPath, pathOr } from 'ramda';
import { generateProfileId, toDots } from '@mixan/common';
import type { IDBProfile } from '@mixan/db';
import { db, getSalts } from '@mixan/db';
import { getProfileById, upsertProfile } from '@mixan/db';
import type {
IncrementProfilePayload,
UpdateProfilePayload,
} from '@mixan/types';
async function findProfile({
profileId,
ip,
origin,
ua,
}: {
profileId: string | null;
ip: string;
origin: string;
ua: string;
}) {
const salts = await getSalts();
const currentProfileId = generateProfileId({
salt: salts.current,
origin,
ip,
ua,
});
const previousProfileId = generateProfileId({
salt: salts.previous,
origin,
ip,
ua,
});
const ids = [currentProfileId, previousProfileId];
if (profileId) {
ids.push(profileId);
}
const profiles = await db.profile.findMany({
where: {
id: {
in: ids,
},
},
});
return profiles.find((p) => {
return (
p.id === profileId ||
p.id === currentProfileId ||
p.id === previousProfileId
);
}) as IDBProfile | undefined;
}
export async function updateProfile(
request: FastifyRequest<{
Body: UpdateProfilePayload;
}>,
reply: FastifyReply
) {
const body = request.body;
const profileId: string | null = body.profileId ?? null;
const { profileId, properties, ...rest } = request.body;
const projectId = request.projectId;
const ip = getClientIp(request)!;
const origin = request.headers.origin ?? projectId;
const ua = request.headers['user-agent']!;
const salts = await getSalts();
const uaInfo = parseUserAgent(ua);
const geo = await parseIp(ip);
if (profileId === null) {
const currentProfileId = generateProfileId({
salt: salts.current,
origin,
ip,
ua,
});
const previousProfileId = generateProfileId({
salt: salts.previous,
origin,
ip,
ua,
});
const profiles = await db.profile.findMany({
where: {
id: {
in: [currentProfileId, previousProfileId],
},
},
});
if (profiles.length === 0) {
const profile = await db.profile.create({
data: {
id: currentProfileId,
external_id: body.id,
first_name: body.first_name,
last_name: body.last_name,
email: body.email,
avatar: body.avatar,
project_id: projectId,
properties: body.properties ?? {},
// ...uaInfo,
// ...geo,
},
});
return reply.status(201).send(profile);
}
const currentProfile = profiles.find((p) => p.id === currentProfileId);
const previousProfile = profiles.find((p) => p.id === previousProfileId);
const profile = currentProfile ?? previousProfile;
if (profile) {
await db.profile.update({
where: {
id: profile.id,
},
data: {
external_id: body.id,
first_name: body.first_name,
last_name: body.last_name,
email: body.email,
avatar: body.avatar,
properties: toDots(
mergeDeepRight(
profile.properties as Record<string, unknown>,
body.properties ?? {}
)
),
// ...uaInfo,
// ...geo,
},
});
return reply.status(200).send(profile.id);
}
return reply.status(200).send();
}
const profile = await db.profile.findUnique({
where: {
id: profileId,
await upsertProfile({
id: profileId,
projectId,
properties: {
...(properties ?? {}),
...(ip ? geo : {}),
...(isUserAgentSet(ua) ? uaInfo : {}),
},
...rest,
});
if (profile) {
await db.profile.update({
where: {
id: profile.id,
},
data: {
external_id: body.id,
first_name: body.first_name,
last_name: body.last_name,
email: body.email,
avatar: body.avatar,
properties: toDots(
mergeDeepRight(
profile.properties as Record<string, unknown>,
body.properties ?? {}
)
),
// ...uaInfo,
// ...geo,
},
});
} else {
await db.profile.create({
data: {
id: profileId,
external_id: body.id,
first_name: body.first_name,
last_name: body.last_name,
email: body.email,
avatar: body.avatar,
project_id: projectId,
properties: body.properties ?? {},
// ...uaInfo,
// ...geo,
},
});
}
reply.status(202).send(profileId);
}
@@ -199,43 +42,33 @@ export async function incrementProfileProperty(
}>,
reply: FastifyReply
) {
const body = request.body;
const profileId: string | null = body.profileId ?? null;
const { profileId, property, value } = request.body;
const projectId = request.projectId;
const ip = getClientIp(request)!;
const origin = request.headers.origin ?? projectId;
const ua = request.headers['user-agent']!;
const profile = await findProfile({
ip,
origin,
ua,
profileId,
});
const profile = await getProfileById(profileId);
if (!profile) {
return reply.status(404).send('Not found');
}
const property = path(body.property.split('.'), profile.properties);
const parsed = parseInt(
pathOr<string>('0', property.split('.'), profile.properties),
10
);
if (typeof property !== 'number' && typeof property !== 'undefined') {
if (isNaN(parsed)) {
return reply.status(400).send('Not number');
}
profile.properties = assocPath(
body.property.split('.'),
property ? property + body.value : body.value,
property.split('.'),
parsed + value,
profile.properties
);
await db.profile.update({
where: {
id: profile.id,
},
data: {
properties: profile.properties as any,
},
await upsertProfile({
id: profile.id,
projectId,
properties: profile.properties,
});
reply.status(202).send(profile.id);
@@ -247,43 +80,33 @@ export async function decrementProfileProperty(
}>,
reply: FastifyReply
) {
const body = request.body;
const profileId: string | null = body.profileId ?? null;
const { profileId, property, value } = request.body;
const projectId = request.projectId;
const ip = getClientIp(request)!;
const origin = request.headers.origin ?? projectId;
const ua = request.headers['user-agent']!;
const profile = await findProfile({
ip,
origin,
ua,
profileId,
});
const profile = await getProfileById(profileId);
if (!profile) {
return reply.status(404).send('Not found');
}
const property = path(body.property.split('.'), profile.properties);
const parsed = parseInt(
pathOr<string>('0', property.split('.'), profile.properties),
10
);
if (typeof property !== 'number') {
if (isNaN(parsed)) {
return reply.status(400).send('Not number');
}
profile.properties = assocPath(
body.property.split('.'),
property ? property - body.value : -body.value,
property.split('.'),
parsed - value,
profile.properties
);
await db.profile.update({
where: {
id: profile.id,
},
data: {
properties: profile.properties as any,
},
await upsertProfile({
id: profile.id,
projectId,
properties: profile.properties,
});
reply.status(202).send(profile.id);

5
apps/sdk-api/src/index.ts
View File

@@ -59,7 +59,10 @@ const startServer = async () => {
}
}
await fastify.listen({ host: '0.0.0.0', port });
await fastify.listen({
host: process.env.NODE_ENV === 'production' ? '0.0.0.0' : 'localhost',
port,
});
// Notify when keys expires
redisPub.config('SET', 'notify-keyspace-events', 'Ex');

4
apps/sdk-api/src/utils/parseUserAgent.ts
View File

@@ -1,5 +1,9 @@
import { UAParser } from 'ua-parser-js';
export function isUserAgentSet(ua: string) {
return ua !== 'node' && ua !== 'undici' && !!ua;
}
export function parseUserAgent(ua: string) {
const res = new UAParser(ua).getResult();
return {