batching events
This commit is contained in:
committed by
Carl-Gerhard Lindesvärd
parent
244aa3b0d3
commit
5e225b7ae6
@@ -6,28 +6,42 @@ import { Worker } from 'bullmq';
|
||||
import express from 'express';
|
||||
|
||||
import { connection, eventsQueue } from '@openpanel/queue';
|
||||
import { cronQueue } from '@openpanel/queue/src/queues';
|
||||
import { cronQueue, sessionsQueue } from '@openpanel/queue/src/queues';
|
||||
|
||||
import { cronJob } from './jobs/cron';
|
||||
import { eventsJob } from './jobs/events';
|
||||
import { sessionsJob } from './jobs/sessions';
|
||||
|
||||
const PORT = parseInt(process.env.WORKER_PORT || '3000', 10);
|
||||
const serverAdapter = new ExpressAdapter();
|
||||
serverAdapter.setBasePath('/');
|
||||
serverAdapter.setBasePath(process.env.SELF_HOSTED ? '/worker' : '/');
|
||||
const app = express();
|
||||
|
||||
const workerOptions: WorkerOptions = {
|
||||
connection,
|
||||
connection: {
|
||||
...connection,
|
||||
enableReadyCheck: false,
|
||||
maxRetriesPerRequest: null,
|
||||
enableOfflineQueue: true,
|
||||
},
|
||||
concurrency: parseInt(process.env.CONCURRENCY || '1', 10),
|
||||
};
|
||||
|
||||
async function start() {
|
||||
new Worker(eventsQueue.name, eventsJob, workerOptions);
|
||||
|
||||
new Worker(cronQueue.name, cronJob, workerOptions);
|
||||
const eventsWorker = new Worker(eventsQueue.name, eventsJob, workerOptions);
|
||||
const sessionsWorker = new Worker(
|
||||
sessionsQueue.name,
|
||||
sessionsJob,
|
||||
workerOptions
|
||||
);
|
||||
const cronWorker = new Worker(cronQueue.name, cronJob, workerOptions);
|
||||
|
||||
createBullBoard({
|
||||
queues: [new BullMQAdapter(eventsQueue), new BullMQAdapter(cronQueue)],
|
||||
queues: [
|
||||
new BullMQAdapter(eventsQueue),
|
||||
new BullMQAdapter(sessionsQueue),
|
||||
new BullMQAdapter(cronQueue),
|
||||
],
|
||||
serverAdapter: serverAdapter,
|
||||
});
|
||||
|
||||
@@ -37,9 +51,61 @@ async function start() {
|
||||
console.log(`For the UI, open http://localhost:${PORT}/`);
|
||||
});
|
||||
|
||||
const repeatableJobs = await cronQueue.getRepeatableJobs();
|
||||
function workerLogger(worker: string, type: string, err?: Error) {
|
||||
console.log(`Worker ${worker} -> ${type}`);
|
||||
if (err) {
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(repeatableJobs);
|
||||
const workers = [sessionsWorker, eventsWorker, cronWorker];
|
||||
workers.forEach((worker) => {
|
||||
worker.on('error', (err) => {
|
||||
workerLogger(worker.name, 'error', err);
|
||||
});
|
||||
|
||||
worker.on('closed', () => {
|
||||
workerLogger(worker.name, 'closed');
|
||||
});
|
||||
|
||||
worker.on('ready', () => {
|
||||
workerLogger(worker.name, 'ready');
|
||||
});
|
||||
});
|
||||
|
||||
async function exitHandler(evtOrExitCodeOrError: number | string | Error) {
|
||||
try {
|
||||
await eventsWorker.close();
|
||||
await sessionsWorker.close();
|
||||
await cronWorker.close();
|
||||
} catch (e) {
|
||||
console.error('EXIT HANDLER ERROR', e);
|
||||
}
|
||||
|
||||
process.exit(isNaN(+evtOrExitCodeOrError) ? 1 : +evtOrExitCodeOrError);
|
||||
}
|
||||
|
||||
[
|
||||
'beforeExit',
|
||||
'uncaughtException',
|
||||
'unhandledRejection',
|
||||
'SIGHUP',
|
||||
'SIGINT',
|
||||
'SIGQUIT',
|
||||
'SIGILL',
|
||||
'SIGTRAP',
|
||||
'SIGABRT',
|
||||
'SIGBUS',
|
||||
'SIGFPE',
|
||||
'SIGUSR1',
|
||||
'SIGSEGV',
|
||||
'SIGUSR2',
|
||||
'SIGTERM',
|
||||
].forEach((evt) =>
|
||||
process.on(evt, (evt) => {
|
||||
exitHandler(evt);
|
||||
})
|
||||
);
|
||||
|
||||
await cronQueue.add(
|
||||
'salt',
|
||||
@@ -55,6 +121,43 @@ async function start() {
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
await cronQueue.add(
|
||||
'flush',
|
||||
{
|
||||
type: 'flushEvents',
|
||||
payload: undefined,
|
||||
},
|
||||
{
|
||||
jobId: 'flushEvents',
|
||||
repeat: {
|
||||
every: process.env.BATCH_INTERVAL
|
||||
? parseInt(process.env.BATCH_INTERVAL, 10)
|
||||
: 1000 * 10,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
await cronQueue.add(
|
||||
'flush',
|
||||
{
|
||||
type: 'flushProfiles',
|
||||
payload: undefined,
|
||||
},
|
||||
{
|
||||
jobId: 'flushProfiles',
|
||||
repeat: {
|
||||
every: process.env.BATCH_INTERVAL
|
||||
? parseInt(process.env.BATCH_INTERVAL, 10)
|
||||
: 1000 * 10,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
const repeatableJobs = await cronQueue.getRepeatableJobs();
|
||||
|
||||
console.log('Repeatable jobs:');
|
||||
console.log(repeatableJobs);
|
||||
}
|
||||
|
||||
start();
|
||||
|
||||
@@ -2,7 +2,7 @@ import { generateSalt } from '@openpanel/common';
|
||||
import { db, getCurrentSalt } from '@openpanel/db';
|
||||
|
||||
export async function salt() {
|
||||
const oldSalt = await getCurrentSalt();
|
||||
const oldSalt = await getCurrentSalt().catch(() => null);
|
||||
const newSalt = await db.salt.create({
|
||||
data: {
|
||||
salt: generateSalt(),
|
||||
@@ -13,7 +13,7 @@ export async function salt() {
|
||||
await db.salt.deleteMany({
|
||||
where: {
|
||||
salt: {
|
||||
notIn: [newSalt.salt, oldSalt],
|
||||
notIn: oldSalt ? [newSalt.salt, oldSalt] : [newSalt.salt],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Job } from 'bullmq';
|
||||
|
||||
import { eventBuffer, profileBuffer } from '@openpanel/db';
|
||||
import type { CronQueuePayload } from '@openpanel/queue/src/queues';
|
||||
|
||||
import { salt } from './cron.salt';
|
||||
@@ -9,5 +10,11 @@ export async function cronJob(job: Job<CronQueuePayload>) {
|
||||
case 'salt': {
|
||||
return await salt();
|
||||
}
|
||||
case 'flushEvents': {
|
||||
return await eventBuffer.flush();
|
||||
}
|
||||
case 'flushProfiles': {
|
||||
return await profileBuffer.flush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,26 @@
|
||||
import type { Job } from 'bullmq';
|
||||
|
||||
import { getTime } from '@openpanel/common';
|
||||
import { createEvent, getEvents } from '@openpanel/db';
|
||||
import { createEvent, eventBuffer, getEvents } from '@openpanel/db';
|
||||
import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue/src/queues';
|
||||
|
||||
export async function createSessionEnd(
|
||||
job: Job<EventsQueuePayloadCreateSessionEnd>
|
||||
) {
|
||||
const payload = job.data.payload;
|
||||
const eventsInBuffer = await eventBuffer.findMany(
|
||||
(item) => item.event.session_id === payload.sessionId
|
||||
);
|
||||
|
||||
const sql = `
|
||||
SELECT * FROM events
|
||||
WHERE
|
||||
device_id = '${payload.deviceId}'
|
||||
session_id = '${payload.sessionId}'
|
||||
AND created_at >= (
|
||||
SELECT created_at
|
||||
FROM events
|
||||
WHERE
|
||||
device_id = '${payload.deviceId}'
|
||||
session_id = '${payload.sessionId}'
|
||||
AND name = 'session_start'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
@@ -25,7 +28,11 @@ export async function createSessionEnd(
|
||||
ORDER BY created_at DESC
|
||||
`;
|
||||
job.log(sql);
|
||||
const events = await getEvents(sql);
|
||||
const eventsInDb = await getEvents(sql);
|
||||
// sort last inserted first
|
||||
const events = [...eventsInBuffer, ...eventsInDb].sort(
|
||||
(a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
|
||||
);
|
||||
|
||||
events.map((event, index) => {
|
||||
job.log(
|
||||
@@ -64,7 +71,7 @@ export async function createSessionEnd(
|
||||
},
|
||||
name: 'session_end',
|
||||
duration: sessionDuration,
|
||||
path: lastEvent.path,
|
||||
path: screenViews[0]?.path ?? '',
|
||||
createdAt: new Date(getTime(lastEvent?.createdAt) + 100),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,20 +1,30 @@
|
||||
import { logger } from '@/utils/logger';
|
||||
import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
|
||||
import { isUserAgentSet, parseUserAgent } from '@/utils/parse-user-agent';
|
||||
import { parseUserAgent } from '@/utils/parse-user-agent';
|
||||
import { isSameDomain, parsePath } from '@/utils/url';
|
||||
import type { Job, JobsOptions } from 'bullmq';
|
||||
import type { Job } from 'bullmq';
|
||||
import { omit } from 'ramda';
|
||||
import { escape } from 'sqlstring';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
import { getTime, toISOString } from '@openpanel/common';
|
||||
import type { IServiceCreateEventPayload } from '@openpanel/db';
|
||||
import { createEvent, getEvents } from '@openpanel/db';
|
||||
import { createEvent } from '@openpanel/db';
|
||||
import { getLastScreenViewFromProfileId } from '@openpanel/db/src/services/event.service';
|
||||
import { findJobByPrefix } from '@openpanel/queue';
|
||||
import { eventsQueue } from '@openpanel/queue/src/queues';
|
||||
import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue/src/queues';
|
||||
import { eventsQueue, sessionsQueue } from '@openpanel/queue/src/queues';
|
||||
import type {
|
||||
EventsQueuePayloadCreateSessionEnd,
|
||||
EventsQueuePayloadIncomingEvent,
|
||||
} from '@openpanel/queue/src/queues';
|
||||
import { redis } from '@openpanel/redis';
|
||||
|
||||
function noDateInFuture(eventDate: Date): Date {
|
||||
if (eventDate > new Date()) {
|
||||
return new Date();
|
||||
} else {
|
||||
return eventDate;
|
||||
}
|
||||
}
|
||||
|
||||
const GLOBAL_PROPERTIES = ['__path', '__referrer'];
|
||||
const SESSION_TIMEOUT = 1000 * 60 * 30;
|
||||
const SESSION_END_TIMEOUT = SESSION_TIMEOUT + 1000;
|
||||
@@ -27,12 +37,8 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
projectId,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
// TODO: Remove after 2024-09-26
|
||||
currentDeviceIdDeprecated,
|
||||
previousDeviceIdDeprecated,
|
||||
priority,
|
||||
} = job.data.payload;
|
||||
let deviceId: string | null = null;
|
||||
|
||||
const properties = body.properties ?? {};
|
||||
const getProperty = (name: string): string | undefined => {
|
||||
// replace thing is just for older sdks when we didn't have `__`
|
||||
@@ -44,22 +50,22 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
| undefined) ?? undefined
|
||||
);
|
||||
};
|
||||
const { ua } = headers;
|
||||
const profileId = body.profileId ?? '';
|
||||
const createdAt = new Date(body.timestamp);
|
||||
|
||||
const profileId = body.profileId ? String(body.profileId) : '';
|
||||
const createdAt = noDateInFuture(new Date(body.timestamp));
|
||||
const url = getProperty('__path');
|
||||
const { path, hash, query, origin } = parsePath(url);
|
||||
const referrer = isSameDomain(getProperty('__referrer'), url)
|
||||
? null
|
||||
: parseReferrer(getProperty('__referrer'));
|
||||
const utmReferrer = getReferrerWithQuery(query);
|
||||
const uaInfo = ua ? parseUserAgent(ua) : null;
|
||||
const isServerEvent = ua ? !isUserAgentSet(ua) : true;
|
||||
const uaInfo = parseUserAgent(headers.ua);
|
||||
|
||||
if (isServerEvent) {
|
||||
const [event] = await getEvents(
|
||||
`SELECT * FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`
|
||||
);
|
||||
if (uaInfo.isServer) {
|
||||
const event = await getLastScreenViewFromProfileId({
|
||||
profileId,
|
||||
projectId,
|
||||
});
|
||||
|
||||
const payload: Omit<IServiceCreateEventPayload, 'id'> = {
|
||||
name: body.name,
|
||||
@@ -67,7 +73,10 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
sessionId: event?.sessionId || '',
|
||||
profileId,
|
||||
projectId,
|
||||
properties: Object.assign({}, omit(GLOBAL_PROPERTIES, properties)),
|
||||
properties: {
|
||||
...omit(GLOBAL_PROPERTIES, properties),
|
||||
user_agent: headers.ua,
|
||||
},
|
||||
createdAt,
|
||||
country: event?.country || geo.country || '',
|
||||
city: event?.city || geo.city || '',
|
||||
@@ -78,7 +87,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
osVersion: event?.osVersion ?? '',
|
||||
browser: event?.browser ?? '',
|
||||
browserVersion: event?.browserVersion ?? '',
|
||||
device: event?.device ?? '',
|
||||
device: event?.device ?? uaInfo.device ?? '',
|
||||
brand: event?.brand ?? '',
|
||||
model: event?.model ?? '',
|
||||
duration: 0,
|
||||
@@ -94,82 +103,50 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
return createEvent(payload);
|
||||
}
|
||||
|
||||
const [sessionEndKeys, eventsKeys] = await Promise.all([
|
||||
redis.keys(`bull:events:sessionEnd:${projectId}:*`),
|
||||
redis.keys(`bull:events:event:${projectId}:*`),
|
||||
]);
|
||||
const sessionEnd = await getSessionEndWithPriority(priority)({
|
||||
projectId,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
});
|
||||
|
||||
const sessionEndJobCurrentDeviceId = await findJobByPrefix(
|
||||
eventsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${currentDeviceId}:`
|
||||
);
|
||||
const sessionEndJobPreviousDeviceId = await findJobByPrefix(
|
||||
eventsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${previousDeviceId}:`
|
||||
);
|
||||
// TODO: Remove after 2024-09-26
|
||||
const sessionEndJobCurrentDeviceIdDeprecated = await findJobByPrefix(
|
||||
eventsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${currentDeviceIdDeprecated}:`
|
||||
);
|
||||
const sessionEndJobPreviousDeviceIdDeprecated = await findJobByPrefix(
|
||||
eventsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${previousDeviceIdDeprecated}:`
|
||||
);
|
||||
const sessionEndPayload = (sessionEnd?.job?.data
|
||||
?.payload as EventsQueuePayloadCreateSessionEnd['payload']) || {
|
||||
sessionId: uuid(),
|
||||
deviceId: currentDeviceId,
|
||||
profileId,
|
||||
};
|
||||
|
||||
let createSessionStart = false;
|
||||
const sessionEndJobId =
|
||||
sessionEnd?.job.id ??
|
||||
`sessionEnd:${projectId}:${sessionEndPayload.deviceId}:${Date.now()}`;
|
||||
|
||||
if (sessionEndJobCurrentDeviceId) {
|
||||
deviceId = currentDeviceId;
|
||||
sessionEndJobCurrentDeviceId.changeDelay(SESSION_END_TIMEOUT);
|
||||
} else if (sessionEndJobPreviousDeviceId) {
|
||||
deviceId = previousDeviceId;
|
||||
sessionEndJobPreviousDeviceId.changeDelay(SESSION_END_TIMEOUT);
|
||||
} else if (sessionEndJobCurrentDeviceIdDeprecated) {
|
||||
deviceId = currentDeviceIdDeprecated;
|
||||
sessionEndJobCurrentDeviceIdDeprecated.changeDelay(SESSION_END_TIMEOUT);
|
||||
} else if (sessionEndJobPreviousDeviceIdDeprecated) {
|
||||
deviceId = previousDeviceIdDeprecated;
|
||||
sessionEndJobPreviousDeviceIdDeprecated.changeDelay(SESSION_END_TIMEOUT);
|
||||
if (sessionEnd) {
|
||||
// If for some reason we have a session end job that is not a createSessionEnd job
|
||||
if (sessionEnd.job.data.type !== 'createSessionEnd') {
|
||||
throw new Error('Invalid session end job');
|
||||
}
|
||||
|
||||
await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
|
||||
} else {
|
||||
deviceId = currentDeviceId;
|
||||
createSessionStart = true;
|
||||
// Queue session end
|
||||
eventsQueue.add(
|
||||
'event',
|
||||
await sessionsQueue.add(
|
||||
'session',
|
||||
{
|
||||
type: 'createSessionEnd',
|
||||
payload: {
|
||||
deviceId,
|
||||
},
|
||||
payload: sessionEndPayload,
|
||||
},
|
||||
{
|
||||
delay: SESSION_END_TIMEOUT,
|
||||
jobId: `sessionEnd:${projectId}:${deviceId}:${Date.now()}`,
|
||||
jobId: sessionEndJobId,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const prevEventJob = await findJobByPrefix(
|
||||
eventsQueue,
|
||||
eventsKeys,
|
||||
`event:${projectId}:${deviceId}:`
|
||||
);
|
||||
|
||||
const [sessionStartEvent] = await getEvents(
|
||||
`SELECT * FROM events WHERE name = 'session_start' AND device_id = ${escape(deviceId)} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`
|
||||
);
|
||||
|
||||
const payload: Omit<IServiceCreateEventPayload, 'id'> = {
|
||||
name: body.name,
|
||||
deviceId,
|
||||
deviceId: sessionEndPayload.deviceId,
|
||||
sessionId: sessionEndPayload.sessionId,
|
||||
profileId,
|
||||
projectId,
|
||||
sessionId: createSessionStart ? uuid() : sessionStartEvent?.sessionId ?? '',
|
||||
properties: Object.assign({}, omit(GLOBAL_PROPERTIES, properties), {
|
||||
__hash: hash,
|
||||
__query: query,
|
||||
@@ -189,7 +166,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
model: uaInfo?.model ?? '',
|
||||
duration: 0,
|
||||
path: path,
|
||||
origin: origin || sessionStartEvent?.origin || '',
|
||||
origin: origin,
|
||||
referrer: referrer?.url,
|
||||
referrerName: referrer?.name || utmReferrer?.name || '',
|
||||
referrerType: referrer?.type || utmReferrer?.type || '',
|
||||
@@ -197,76 +174,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
meta: undefined,
|
||||
};
|
||||
|
||||
const isDelayed = prevEventJob ? await prevEventJob?.isDelayed() : false;
|
||||
|
||||
if (isDelayed && prevEventJob && prevEventJob.data.type === 'createEvent') {
|
||||
const prevEvent = prevEventJob.data.payload;
|
||||
const duration = getTime(payload.createdAt) - getTime(prevEvent.createdAt);
|
||||
job.log(`prevEvent ${JSON.stringify(prevEvent, null, 2)}`);
|
||||
|
||||
// Set path from prev screen_view event if current event is not a screen_view
|
||||
if (payload.name != 'screen_view') {
|
||||
payload.path = prevEvent.path;
|
||||
}
|
||||
|
||||
if (payload.name === 'screen_view') {
|
||||
if (duration < 0) {
|
||||
logger.info({ prevEvent, payload }, 'Duration is negative');
|
||||
} else {
|
||||
try {
|
||||
// Skip update duration if it's wrong
|
||||
// Seems like request is not in right order
|
||||
await prevEventJob.updateData({
|
||||
type: 'createEvent',
|
||||
payload: {
|
||||
...prevEvent,
|
||||
duration,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{
|
||||
error,
|
||||
prevEventJobStatus: await prevEventJob
|
||||
.getState()
|
||||
.catch(() => 'unknown'),
|
||||
},
|
||||
`Failed update delayed job`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await prevEventJob.promote();
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
{
|
||||
error,
|
||||
prevEventJobStatus: await prevEventJob
|
||||
.getState()
|
||||
.catch(() => 'unknown'),
|
||||
prevEvent,
|
||||
currEvent: payload,
|
||||
},
|
||||
`Failed to promote job`
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if (payload.name !== 'screen_view') {
|
||||
job.log(
|
||||
`no previous job ${JSON.stringify(
|
||||
{
|
||||
prevEventJob,
|
||||
payload,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
if (createSessionStart) {
|
||||
// We do not need to queue session_start
|
||||
if (!sessionEnd) {
|
||||
await createEvent({
|
||||
...payload,
|
||||
name: 'session_start',
|
||||
@@ -275,40 +183,78 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
|
||||
});
|
||||
}
|
||||
|
||||
const options: JobsOptions = {};
|
||||
if (payload.name === 'screen_view') {
|
||||
options.delay = SESSION_TIMEOUT;
|
||||
options.jobId = `event:${projectId}:${deviceId}:${Date.now()}`;
|
||||
return createEvent(payload);
|
||||
}
|
||||
|
||||
function getSessionEndWithPriority(
|
||||
priority: boolean,
|
||||
count = 0
|
||||
): typeof getSessionEnd {
|
||||
return async (args) => {
|
||||
const res = await getSessionEnd(args);
|
||||
|
||||
if (count > 5) {
|
||||
throw new Error('Failed to get session end');
|
||||
}
|
||||
|
||||
// if we get simultaneous requests we want to avoid race conditions with getting the session end
|
||||
// one of the events will get priority and the other will wait for the first to finish
|
||||
if (res === null && priority === false) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
return getSessionEndWithPriority(priority, count + 1)(args);
|
||||
}
|
||||
|
||||
return res;
|
||||
};
|
||||
}
|
||||
|
||||
async function getSessionEnd({
|
||||
projectId,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
}: {
|
||||
projectId: string;
|
||||
currentDeviceId: string;
|
||||
previousDeviceId: string;
|
||||
}) {
|
||||
const sessionEndKeys = await redis.keys(`*:sessionEnd:${projectId}:*`);
|
||||
|
||||
const sessionEndJobCurrentDeviceId = await findJobByPrefix(
|
||||
sessionsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${currentDeviceId}:`
|
||||
);
|
||||
if (sessionEndJobCurrentDeviceId) {
|
||||
return { deviceId: currentDeviceId, job: sessionEndJobCurrentDeviceId };
|
||||
}
|
||||
|
||||
job.log(
|
||||
`event is queued ${JSON.stringify(
|
||||
{
|
||||
ua,
|
||||
uaInfo,
|
||||
referrer,
|
||||
profileId,
|
||||
projectId,
|
||||
deviceId,
|
||||
geo,
|
||||
sessionStartEvent,
|
||||
path,
|
||||
payload,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
const sessionEndJobCurrentDeviceId2 = await findJobByPrefix(
|
||||
eventsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${currentDeviceId}:`
|
||||
);
|
||||
if (sessionEndJobCurrentDeviceId2) {
|
||||
return { deviceId: currentDeviceId, job: sessionEndJobCurrentDeviceId2 };
|
||||
}
|
||||
|
||||
// Queue event instead of creating it,
|
||||
// since we want to update duration if we get more events in the same session
|
||||
// The event will only be delayed if it's a screen_view event
|
||||
return eventsQueue.add(
|
||||
'event',
|
||||
{
|
||||
type: 'createEvent',
|
||||
payload,
|
||||
},
|
||||
options
|
||||
const sessionEndJobPreviousDeviceId = await findJobByPrefix(
|
||||
sessionsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${previousDeviceId}:`
|
||||
);
|
||||
if (sessionEndJobPreviousDeviceId) {
|
||||
return { deviceId: previousDeviceId, job: sessionEndJobPreviousDeviceId };
|
||||
}
|
||||
|
||||
const sessionEndJobPreviousDeviceId2 = await findJobByPrefix(
|
||||
eventsQueue,
|
||||
sessionEndKeys,
|
||||
`sessionEnd:${projectId}:${previousDeviceId}:`
|
||||
);
|
||||
if (sessionEndJobPreviousDeviceId2) {
|
||||
return { deviceId: previousDeviceId, job: sessionEndJobPreviousDeviceId2 };
|
||||
}
|
||||
|
||||
// Create session
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { Job } from 'bullmq';
|
||||
import { escape } from 'sqlstring';
|
||||
|
||||
import { chQuery, createEvent, db } from '@openpanel/db';
|
||||
import { chQuery, db } from '@openpanel/db';
|
||||
import type {
|
||||
EventsQueuePayload,
|
||||
EventsQueuePayloadCreateSessionEnd,
|
||||
@@ -16,22 +16,6 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {
|
||||
case 'incomingEvent': {
|
||||
return await incomingEvent(job as Job<EventsQueuePayloadIncomingEvent>);
|
||||
}
|
||||
case 'createEvent': {
|
||||
if (job.attemptsStarted > 1 && job.data.payload.duration < 0) {
|
||||
job.data.payload.duration = 0;
|
||||
}
|
||||
const createdEvent = await createEvent(job.data.payload);
|
||||
try {
|
||||
await updateEventsCount(job.data.payload.projectId);
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
job.log(`Failed to update events count: ${e.message}`);
|
||||
} else {
|
||||
job.log(`Failed to update events count: Unknown issue`);
|
||||
}
|
||||
}
|
||||
return createdEvent;
|
||||
}
|
||||
case 'createSessionEnd': {
|
||||
return await createSessionEnd(
|
||||
job as Job<EventsQueuePayloadCreateSessionEnd>
|
||||
|
||||
9
apps/worker/src/jobs/sessions.ts
Normal file
9
apps/worker/src/jobs/sessions.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import type { Job } from 'bullmq';
|
||||
|
||||
import type { SessionsQueuePayload } from '@openpanel/queue/src/queues';
|
||||
|
||||
import { createSessionEnd } from './events.create-session-end';
|
||||
|
||||
export async function sessionsJob(job: Job<SessionsQueuePayload>) {
|
||||
return await createSessionEnd(job);
|
||||
}
|
||||
@@ -1,11 +1,16 @@
|
||||
import { UAParser } from 'ua-parser-js';
|
||||
|
||||
export function isUserAgentSet(ua: string) {
|
||||
return ua !== 'node' && ua !== 'undici' && !!ua;
|
||||
}
|
||||
const parsedServerUa = {
|
||||
isServer: true,
|
||||
device: 'server',
|
||||
} as const;
|
||||
|
||||
export function parseUserAgent(ua: string) {
|
||||
export function parseUserAgent(ua?: string | null) {
|
||||
if (!ua) return parsedServerUa;
|
||||
const res = new UAParser(ua).getResult();
|
||||
|
||||
if (isServer(ua)) return parsedServerUa;
|
||||
|
||||
return {
|
||||
os: res.os.name,
|
||||
osVersion: res.os.version,
|
||||
@@ -14,20 +19,86 @@ export function parseUserAgent(ua: string) {
|
||||
device: res.device.type ?? getDevice(ua),
|
||||
brand: res.device.vendor,
|
||||
model: res.device.model,
|
||||
};
|
||||
isServer: false,
|
||||
} as const;
|
||||
}
|
||||
|
||||
const userAgentServerList: string[] = [
|
||||
// Node.js libraries
|
||||
'node',
|
||||
'node-fetch',
|
||||
'axios',
|
||||
'request',
|
||||
'superagent',
|
||||
'undici',
|
||||
|
||||
// Python libraries
|
||||
'python-requests',
|
||||
'python-urllib',
|
||||
|
||||
// Ruby libraries
|
||||
'Faraday',
|
||||
'Ruby',
|
||||
'http.rb',
|
||||
|
||||
// Go libraries
|
||||
'Go-http-client',
|
||||
'Go-http-client',
|
||||
|
||||
// Java libraries
|
||||
'Apache-HttpClient',
|
||||
'okhttp',
|
||||
'okhowtp',
|
||||
|
||||
// PHP libraries
|
||||
'GuzzleHttp',
|
||||
'PHP-cURL',
|
||||
|
||||
// Other
|
||||
'Dart',
|
||||
'RestSharp', // Popular .NET HTTP client library
|
||||
'HttpClientFactory', // .NET's typed client factory
|
||||
'Ktor', // A client for Kotlin
|
||||
'Ning', // Async HTTP client for Java
|
||||
'grpc-csharp', // gRPC for C#
|
||||
'Volley', // HTTP library used in Android apps for making network requests
|
||||
'Spring',
|
||||
'vert.x',
|
||||
'grpc-',
|
||||
];
|
||||
|
||||
function isServer(userAgent: string) {
|
||||
const match = userAgentServerList.some((server) =>
|
||||
userAgent.includes(server)
|
||||
);
|
||||
if (match) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return !!userAgent.match(/^[^\s]+\/[\d.]+$/);
|
||||
}
|
||||
|
||||
export function getDevice(ua: string) {
|
||||
const t1 =
|
||||
const mobile1 =
|
||||
/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
|
||||
ua
|
||||
);
|
||||
const t2 =
|
||||
const mobile2 =
|
||||
/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
|
||||
ua.slice(0, 4)
|
||||
);
|
||||
if (t1 || t2) {
|
||||
const tablet =
|
||||
/tablet|ipad|android(?!.*mobile)|xoom|sch-i800|kindle|silk|playbook/i.test(
|
||||
ua
|
||||
);
|
||||
|
||||
if (mobile1 || mobile2) {
|
||||
return 'mobile';
|
||||
}
|
||||
|
||||
if (tablet) {
|
||||
return 'tablet';
|
||||
}
|
||||
|
||||
return 'desktop';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user