zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

batching events

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-07-17 17:13:07 +02:00
committed by Carl-Gerhard Lindesvärd
parent 244aa3b0d3
commit 5e225b7ae6
58 changed files with 2204 additions and 583 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
apps/api/Dockerfile
View File

@@ -1,48 +1,16 @@
# Dockerfile that builds the web app only
FROM --platform=linux/amd64 node:20-slim AS base
ARG NODE_VERSION=20
FROM --platform=linux/amd64 node:${NODE_VERSION}-slim AS base
ARG DATABASE_URL
ENV DATABASE_URL=$DATABASE_URL
ARG CLICKHOUSE_DB
ENV CLICKHOUSE_DB=$CLICKHOUSE_DB
ARG CLICKHOUSE_PASSWORD
ENV CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
ARG CLICKHOUSE_URL
ENV CLICKHOUSE_URL=$CLICKHOUSE_URL
ARG CLICKHOUSE_USER
ENV CLICKHOUSE_USER=$CLICKHOUSE_USER
ARG REDIS_URL
ENV REDIS_URL=$REDIS_URL
ARG NEXT_PUBLIC_DASHBOARD_URL
ENV NEXT_PUBLIC_DASHBOARD_URL=$NEXT_PUBLIC_DASHBOARD_URL
ARG NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
ARG CLERK_SECRET_KEY
ENV CLERK_SECRET_KEY=$CLERK_SECRET_KEY
ARG CLERK_PUBLIC_PEM_KEY
ENV CLERK_PUBLIC_PEM_KEY=$CLERK_PUBLIC_PEM_KEY
ARG SEVENTY_SEVEN_API_KEY
ENV SEVENTY_SEVEN_API_KEY=$SEVENTY_SEVEN_API_KEY
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
ARG NODE_VERSION=20
RUN apt update \
&& apt install -y curl python3 make g++ \
&& curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n \
@@ -73,7 +41,7 @@ WORKDIR /app/apps/api
RUN pnpm install --frozen-lockfile
WORKDIR /app
COPY apps apps
COPY apps/api apps/api
COPY packages packages
COPY tooling tooling
RUN pnpm db:codegen

1
apps/api/package.json
View File

@@ -35,6 +35,7 @@
"sharp": "^0.33.2",
"sqlstring": "^2.3.3",
"superjson": "^1.13.3",
"svix": "^1.24.0",
"ua-parser-js": "^1.0.37",
"url-metadata": "^4.1.0",
"zod": "^3.22.4"

189
apps/api/scripts/get-organizations.ts Normal file
View File

@@ -0,0 +1,189 @@
// import { clerkClient } from '@clerk/fastify';
import { db } from '@openpanel/db';
// import { db } from '@openpanel/db';
// type Fn<T = unknown> = (args: { limit: number; offset: number }) => Promise<{
// data: T[];
// totalCount: number;
// }>;
// function getAllDataByPagination<T extends Fn>(
// cb: T
// ): Promise<Awaited<ReturnType<T>>['data']> {
// const data: Awaited<ReturnType<T>>['data'] = [];
// async function getData(page = 0) {
// console.log(`getData with offset ${page * 100}`);
// const response = await cb({
// limit: 100,
// offset: page * 100,
// });
// if (response.data.length !== 0) {
// data.push(...response.data);
// await getData(page + 1);
// }
// await new Promise((resolve) => setTimeout(resolve, 100));
// }
// return getData().then(() => data);
// }
// async function main() {
// const organizations = await getAllDataByPagination(
// clerkClient.organizations.getOrganizationList.bind(
// clerkClient.organizations
// )
// );
// const users = await getAllDataByPagination(
// clerkClient.users.getUserList.bind(clerkClient.users)
// );
// console.log(`Found ${organizations.length} organizations`);
// console.log(`Found ${users.length} users`);
// for (const user of users.slice(-10)) {
// const email = user.primaryEmailAddress?.emailAddress;
// console.log('Check', email);
// try {
// if (email) {
// const exists = await db.user.findUnique({
// where: {
// id: user.id,
// },
// });
// if (exists) {
// console.log('already exists');
// } else {
// await db.user.create({
// data: {
// id: user.id,
// email: email,
// firstName: user.firstName,
// lastName: user.lastName,
// },
// });
// }
// } else {
// console.log('No email?', user);
// }
// } catch (e) {
// console.log('ERROR');
// console.log('');
// console.log('');
// console.dir(user, { depth: null });
// console.log('');
// console.log('');
// console.log('');
// }
// }
// for (const org of organizations.slice(-20)) {
// try {
// if (org.slug) {
// const exists = await db.organization.findUnique({
// where: {
// id: org.slug,
// },
// });
// if (exists) {
// console.log('already exists org');
// } else {
// const clerkOrgMembers =
// await clerkClient.organizations.getOrganizationMembershipList({
// organizationId: org.id,
// });
// const members = clerkOrgMembers.data.map((member) => {
// const user = users.find(
// (u) => u.id === member.publicUserData?.userId
// );
// return {
// userId: member.publicUserData?.userId,
// role: member.role,
// email: user!.primaryEmailAddress!.emailAddress,
// };
// });
// await db.organization.create({
// data: {
// id: org.slug,
// name: org.name,
// createdBy: {
// connect: {
// id: org.createdBy,
// },
// },
// members: {
// create: members,
// },
// },
// });
// const invites =
// await clerkClient.organizations.getOrganizationInvitationList({
// organizationId: org.id,
// status: ['pending'],
// });
// for (const invite of invites.data) {
// await db.member.create({
// data: {
// email: invite.emailAddress,
// organizationId: org.slug,
// role: invite.role,
// userId: null,
// meta: {
// access: invite.publicMetadata?.access as string[],
// invitationId: invite.id,
// },
// },
// });
// }
// }
// } else {
// console.log('org does not have any slug', org);
// }
// } catch (e) {
// console.log('ERROR');
// console.log('');
// console.log('');
// console.dir(org, { depth: null });
// console.log('');
// console.log('');
// console.log('');
// }
// }
// process.exit(0);
// }
// main();
async function main() {
const organization = await db.organization.findUnique({
where: {
id: 'openpanel-dev',
members: {
some: {
userId: 'user_2cEoI8b1SuEFbZERGEAyVvC676F',
},
},
},
include: {
members: {
select: {
role: true,
user: true,
},
},
},
});
console.dir(organization, { depth: null });
}
main();

65
apps/api/scripts/migrate-origins.ts Normal file
View File

@@ -0,0 +1,65 @@
import { ch, chQuery } from '@openpanel/db';
async function main() {
const projects = await chQuery(
`SELECT distinct project_id FROM events ORDER BY project_id`
);
const withOrigin = [];
for (const project of projects) {
try {
const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([
await chQuery(
`SELECT * FROM events WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`
),
await chQuery(
`SELECT * FROM events WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`
),
]);
if (eventWithOrigin[0] && eventWithoutOrigin[0]) {
console.log(`Project ${project.project_id} as events without origin`);
console.log(`- Origin: ${eventWithOrigin[0].origin}`);
withOrigin.push(project.project_id);
const events = await chQuery(
`SELECT count(*) as count FROM events WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`
);
console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);
await ch.command({
query: `ALTER TABLE events UPDATE origin = '${eventWithOrigin[0].origin}' WHERE project_id = '${project.project_id}' AND path != '' AND origin = '';`,
clickhouse_settings: {
wait_end_of_query: 1,
},
});
}
if (!eventWithOrigin[0] && eventWithoutOrigin[0]) {
console.log(
`😧 Project ${project.project_id} has no events with origin (last event ${eventWithoutOrigin[0].created_at})`
);
console.log('- NO ORIGIN');
}
if (!eventWithOrigin[0] && !eventWithoutOrigin[0]) {
console.log(
`🔥 WARNING: Project ${project.project_id} has no events at all?!?!?!`
);
}
if (eventWithOrigin[0] && !eventWithoutOrigin[0]) {
console.log(
`✅ Project ${project.project_id} has all events with origin!!!`
);
}
console.log('');
console.log('');
await new Promise((resolve) => setTimeout(resolve, 500));
} catch (e) {
console.log('🥵 ERROR ORRROR');
console.log('Error for project', project.project_id);
}
}
process.exit(0);
}
main();

34
apps/api/src/controllers/event.controller.ts
View File

@@ -5,6 +5,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId } from '@openpanel/common';
import { getSalts } from '@openpanel/db';
import { eventsQueue } from '@openpanel/queue';
import { redis } from '@openpanel/redis';
import type { PostEventPayload } from '@openpanel/sdk';
export async function postEvent(
@@ -63,7 +64,6 @@ export async function postEvent(
}
const ip = getClientIp(request)!;
const ua = request.headers['user-agent']!;
const origin = request.headers.origin!;
const projectId = request.client?.projectId;
if (!projectId) {
@@ -84,19 +84,15 @@ export async function postEvent(
ip,
ua,
});
// TODO: Remove after 2024-09-26
const currentDeviceIdDeprecated = generateDeviceId({
salt: salts.current,
origin,
ip,
ua,
});
const previousDeviceIdDeprecated = generateDeviceId({
salt: salts.previous,
origin,
ip,
ua,
});
// this will ensure that we don't have multiple events creating sessions
const locked = await redis.set(
`request:priority:${currentDeviceId}-${previousDeviceId}`,
'locked',
'EX',
10,
'NX'
);
eventsQueue.add('event', {
type: 'incomingEvent',
@@ -105,13 +101,15 @@ export async function postEvent(
headers: {
ua,
},
event: request.body,
event: {
...request.body,
// Dont rely on the client for the timestamp
timestamp: new Date().toISOString(),
},
geo,
currentDeviceId,
previousDeviceId,
// TODO: Remove after 2024-09-26
currentDeviceIdDeprecated,
previousDeviceIdDeprecated,
priority: locked === 'OK',
},
});

42
apps/api/src/controllers/live.controller.ts
View File

@@ -19,7 +19,7 @@ export function getLiveEventInfo(key: string) {
return key.split(':').slice(2) as [string, string];
}
export async function test(
export async function testVisitors(
req: FastifyRequest<{
Params: {
projectId: string;
@@ -28,13 +28,15 @@ export async function test(
reply: FastifyReply
) {
const events = await getEvents(
`SELECT * FROM events WHERE project_id = ${escape(req.params.projectId)} AND name = 'screen_view' LIMIT 500`
`SELECT * FROM events LIMIT 500`
// `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
);
const event = events[Math.floor(Math.random() * events.length)];
if (!event) {
return reply.status(404).send('No event found');
}
redisPub.publish('event', superjson.stringify(event));
event.projectId = req.params.projectId;
redisPub.publish('event:received', superjson.stringify(event));
redis.set(
`live:event:${event.projectId}:${Math.random() * 1000}`,
'',
@@ -44,6 +46,26 @@ export async function test(
reply.status(202).send(event);
}
export async function testEvents(
req: FastifyRequest<{
Params: {
projectId: string;
};
}>,
reply: FastifyReply
) {
const events = await getEvents(
`SELECT * FROM events LIMIT 500`
// `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
);
const event = events[Math.floor(Math.random() * events.length)];
if (!event) {
return reply.status(404).send('No event found');
}
redisPub.publish('event:saved', superjson.stringify(event));
reply.status(202).send(event);
}
export function wsVisitors(
connection: {
socket: WebSocket;
@@ -56,11 +78,11 @@ export function wsVisitors(
) {
const { params } = req;
redisSub.subscribe('event');
redisSub.subscribe('event:received');
redisSub.psubscribe('__key*:expired');
const message = (channel: string, message: string) => {
if (channel === 'event') {
if (channel === 'event:received') {
const event = getSuperJson<IServiceCreateEventPayload>(message);
if (event?.projectId === params.projectId) {
getLiveVisitors(params.projectId).then((count) => {
@@ -82,7 +104,7 @@ export function wsVisitors(
redisSub.on('pmessage', pmessage);
connection.socket.on('close', () => {
redisSub.unsubscribe('event');
redisSub.unsubscribe('event:saved');
redisSub.punsubscribe('__key*:expired');
redisSub.off('message', message);
redisSub.off('pmessage', pmessage);
@@ -90,7 +112,7 @@ export function wsVisitors(
}
export function wsEvents(connection: { socket: WebSocket }) {
redisSub.subscribe('event');
redisSub.subscribe('event:saved');
const message = (channel: string, message: string) => {
const event = getSuperJson<IServiceCreateEventPayload>(message);
@@ -102,7 +124,7 @@ export function wsEvents(connection: { socket: WebSocket }) {
redisSub.on('message', message);
connection.socket.on('close', () => {
redisSub.unsubscribe('event');
redisSub.unsubscribe('event:saved');
redisSub.off('message', message);
});
}
@@ -129,7 +151,7 @@ export async function wsProjectEvents(
projectId: params.projectId,
});
redisSub.subscribe('event');
redisSub.subscribe('event:saved');
const message = async (channel: string, message: string) => {
const event = getSuperJson<IServiceCreateEventPayload>(message);
@@ -151,7 +173,7 @@ export async function wsProjectEvents(
redisSub.on('message', message as any);
connection.socket.on('close', () => {
redisSub.unsubscribe('event');
redisSub.unsubscribe('event:saved');
redisSub.off('message', message as any);
});
}

4
apps/api/src/controllers/profile.controller.ts
View File

@@ -1,5 +1,5 @@
import { getClientIp, parseIp } from '@/utils/parseIp';
import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
import { parseUserAgent } from '@/utils/parseUserAgent';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, pathOr } from 'ramda';
@@ -29,7 +29,7 @@ export async function updateProfile(
properties: {
...(properties ?? {}),
...(ip ? geo : {}),
...(isUserAgentSet(ua) ? uaInfo : {}),
...uaInfo,
},
...rest,
});

152
apps/api/src/controllers/webhook.controller.ts Normal file
View File

@@ -0,0 +1,152 @@
import type { WebhookEvent } from '@clerk/fastify';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { pathOr } from 'ramda';
import { Webhook } from 'svix';
import { AccessLevel, db } from '@openpanel/db';
if (!process.env.CLERK_SIGNING_SECRET) {
throw new Error('CLERK_SIGNING_SECRET is required');
}
const wh = new Webhook(process.env.CLERK_SIGNING_SECRET);
function verify(body: any, headers: FastifyRequest['headers']) {
try {
const svix_id = headers['svix-id'] as string;
const svix_timestamp = headers['svix-timestamp'] as string;
const svix_signature = headers['svix-signature'] as string;
wh.verify(JSON.stringify(body), {
'svix-id': svix_id,
'svix-timestamp': svix_timestamp,
'svix-signature': svix_signature,
});
return true;
} catch (error) {
return false;
}
}
export async function clerkWebhook(
request: FastifyRequest<{
Body: WebhookEvent;
}>,
reply: FastifyReply
) {
const payload = request.body;
const verified = verify(payload, request.headers);
if (!verified) {
return reply.send({ message: 'Invalid signature' });
}
if (payload.type === 'user.created') {
const email = payload.data.email_addresses[0]?.email_address;
if (!email) {
return Response.json(
{ message: 'No email address found' },
{ status: 400 }
);
}
const user = await db.user.create({
data: {
id: payload.data.id,
email,
firstName: payload.data.first_name,
lastName: payload.data.last_name,
},
});
const memberships = await db.member.findMany({
where: {
email,
userId: null,
},
});
for (const membership of memberships) {
const access = pathOr<string[]>([], ['meta', 'access'], membership);
db.$transaction([
// Update the member to link it to the user
// This will remove the item from invitations
db.member.update({
where: {
id: membership.id,
},
data: {
userId: user.id,
},
}),
db.projectAccess.createMany({
data: access
.filter((a) => typeof a === 'string')
.map((projectId) => ({
organizationSlug: membership.organizationId,
organizationId: membership.organizationId,
projectId: projectId,
userId: user.id,
level: AccessLevel.read,
})),
}),
]);
}
}
if (payload.type === 'organizationMembership.created') {
const access = payload.data.public_metadata.access;
if (Array.isArray(access)) {
await db.projectAccess.createMany({
data: access
.filter((a): a is string => typeof a === 'string')
.map((projectId) => ({
organizationSlug: payload.data.organization.slug,
organizationId: payload.data.organization.slug,
projectId: projectId,
userId: payload.data.public_user_data.user_id,
level: AccessLevel.read,
})),
});
}
}
if (payload.type === 'user.deleted') {
await db.$transaction([
db.user.update({
where: {
id: payload.data.id,
},
data: {
deletedAt: new Date(),
firstName: null,
lastName: null,
email: `deleted+${payload.data.id}@openpanel.dev`,
},
}),
db.projectAccess.deleteMany({
where: {
userId: payload.data.id,
},
}),
db.member.deleteMany({
where: {
userId: payload.data.id,
},
}),
]);
}
if (payload.type === 'organizationMembership.deleted') {
await db.projectAccess.deleteMany({
where: {
organizationSlug: payload.data.organization.slug,
userId: payload.data.public_user_data.user_id,
},
});
}
reply.send({ success: true });
}

68
apps/api/src/index.ts
View File

@@ -19,6 +19,7 @@ import exportRouter from './routes/export.router';
import liveRouter from './routes/live.router';
import miscRouter from './routes/misc.router';
import profileRouter from './routes/profile.router';
import webhookRouter from './routes/webhook.router';
import { logger, logInfo } from './utils/logger';
declare module 'fastify' {
@@ -82,6 +83,7 @@ const startServer = async () => {
fastify.register(profileRouter, { prefix: '/profile' });
fastify.register(miscRouter, { prefix: '/misc' });
fastify.register(exportRouter, { prefix: '/export' });
fastify.register(webhookRouter, { prefix: '/webhook' });
fastify.setErrorHandler((error) => {
logger.error(error, 'Error in request');
});
@@ -89,35 +91,45 @@ const startServer = async () => {
reply.send({ name: 'openpanel sdk api' });
});
fastify.get('/healthcheck', async (request, reply) => {
try {
const redisRes = await withTimings(redis.keys('*'));
const dbRes = await withTimings(db.project.findFirst());
const queueRes = await withTimings(eventsQueue.getCompleted());
const chRes = await withTimings(
chQuery('SELECT * FROM events LIMIT 1')
);
const redisRes = await withTimings(redis.keys('*')).catch(
(e: Error) => e
);
const dbRes = await withTimings(db.project.findFirst()).catch(
(e: Error) => e
);
const queueRes = await withTimings(eventsQueue.getCompleted()).catch(
(e: Error) => e
);
const chRes = await withTimings(
chQuery('SELECT * FROM events LIMIT 1')
).catch((e: Error) => e);
reply.send({
redis: {
ok: redisRes[1].length ? true : false,
time: `${redisRes[0]}ms`,
},
db: {
ok: dbRes[1] ? true : false,
time: `${dbRes[0]}ms`,
},
queue: {
ok: queueRes[1].length ? true : false,
time: `${queueRes[0]}ms`,
},
ch: {
ok: chRes[1].length ? true : false,
time: `${chRes[0]}ms`,
},
});
} catch (e) {
reply.status(500).send();
}
reply.send({
redis: Array.isArray(redisRes)
? {
ok: redisRes[1].length ? true : false,
time: `${redisRes[0]}ms`,
}
: redisRes,
db: Array.isArray(dbRes)
? {
ok: dbRes[1] ? true : false,
time: `${dbRes[0]}ms`,
}
: dbRes,
queue: Array.isArray(queueRes)
? {
ok: queueRes[1].length ? true : false,
time: `${queueRes[0]}ms`,
}
: queueRes,
ch: Array.isArray(chRes)
? {
ok: chRes[1].length ? true : false,
time: `${chRes[0]}ms`,
}
: chRes,
});
});
if (process.env.NODE_ENV === 'production') {
for (const signal of ['SIGINT', 'SIGTERM']) {

7
apps/api/src/routes/live.router.ts
View File

@@ -3,10 +3,15 @@ import fastifyWS from '@fastify/websocket';
import type { FastifyPluginCallback } from 'fastify';
const liveRouter: FastifyPluginCallback = (fastify, opts, done) => {
fastify.route({
method: 'GET',
url: '/visitors/test/:projectId',
handler: controller.testVisitors,
});
fastify.route({
method: 'GET',
url: '/events/test/:projectId',
handler: controller.test,
handler: controller.testEvents,
});
fastify.register(fastifyWS);

13
apps/api/src/routes/webhook.router.ts Normal file
View File

@@ -0,0 +1,13 @@
import * as controller from '@/controllers/webhook.controller';
import type { FastifyPluginCallback } from 'fastify';
const webhookRouter: FastifyPluginCallback = (fastify, opts, done) => {
fastify.route({
method: 'POST',
url: '/clerk',
handler: controller.clerkWebhook,
});
done();
};
export default webhookRouter;

2
apps/api/src/utils/auth.ts
View File

@@ -5,8 +5,6 @@ import { verifyPassword } from '@openpanel/common';
import type { Client, IServiceClient } from '@openpanel/db';
import { ClientType, db } from '@openpanel/db';
import { logger } from './logger';
const cleanDomain = (domain: string) =>
domain
.replace('www.', '')

87
apps/api/src/utils/parseUserAgent.ts
View File

@@ -1,11 +1,16 @@
import { UAParser } from 'ua-parser-js';
export function isUserAgentSet(ua: string) {
return ua !== 'node' && ua !== 'undici' && !!ua;
}
const parsedServerUa = {
isServer: true,
device: 'server',
} as const;
export function parseUserAgent(ua: string) {
export function parseUserAgent(ua?: string | null) {
if (!ua) return parsedServerUa;
const res = new UAParser(ua).getResult();
if (isServer(ua)) return parsedServerUa;
return {
os: res.os.name,
osVersion: res.os.version,
@@ -14,20 +19,86 @@ export function parseUserAgent(ua: string) {
device: res.device.type ?? getDevice(ua),
brand: res.device.vendor,
model: res.device.model,
};
isServer: false,
} as const;
}
const userAgentServerList: string[] = [
// Node.js libraries
'node',
'node-fetch',
'axios',
'request',
'superagent',
'undici',
// Python libraries
'python-requests',
'python-urllib',
// Ruby libraries
'Faraday',
'Ruby',
'http.rb',
// Go libraries
'Go-http-client',
'Go-http-client',
// Java libraries
'Apache-HttpClient',
'okhttp',
'okhowtp',
// PHP libraries
'GuzzleHttp',
'PHP-cURL',
// Other
'Dart',
'RestSharp', // Popular .NET HTTP client library
'HttpClientFactory', // .NET's typed client factory
'Ktor', // A client for Kotlin
'Ning', // Async HTTP client for Java
'grpc-csharp', // gRPC for C#
'Volley', // HTTP library used in Android apps for making network requests
'Spring',
'vert.x',
'grpc-',
];
function isServer(userAgent: string) {
const match = userAgentServerList.some((server) =>
userAgent.includes(server)
);
if (match) {
return true;
}
return !!userAgent.match(/^[^\s]+\/[\d.]+$/);
}
export function getDevice(ua: string) {
const t1 =
const mobile1 =
/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
ua
);
const t2 =
const mobile2 =
/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
ua.slice(0, 4)
);
if (t1 || t2) {
const tablet =
/tablet|ipad|android(?!.*mobile)|xoom|sch-i800|kindle|silk|playbook/i.test(
ua
);
if (mobile1 || mobile2) {
return 'mobile';
}
if (tablet) {
return 'tablet';
}
return 'desktop';
}

49
apps/dashboard/Dockerfile
View File

@@ -1,37 +1,14 @@
FROM --platform=linux/amd64 node:20-slim AS base
ARG NODE_VERSION=20
ARG NEXT_PUBLIC_DASHBOARD_URL
ENV NEXT_PUBLIC_DASHBOARD_URL=$NEXT_PUBLIC_DASHBOARD_URL
FROM --platform=linux/amd64 node:${NODE_VERSION}-slim AS base
ARG NEXT_PUBLIC_API_URL
ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
ENV SKIP_ENV_VALIDATION="1"
ARG DATABASE_URL
ENV DATABASE_URL=$DATABASE_URL
ARG CLICKHOUSE_DB
ENV CLICKHOUSE_DB=$CLICKHOUSE_DB
ARG CLICKHOUSE_PASSWORD
ENV CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
ARG CLICKHOUSE_URL
ENV CLICKHOUSE_URL=$CLICKHOUSE_URL
ARG CLICKHOUSE_USER
ENV CLICKHOUSE_USER=$CLICKHOUSE_USER
ARG REDIS_URL
ENV REDIS_URL=$REDIS_URL
ARG NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
ARG CLERK_SECRET_KEY
ENV CLERK_SECRET_KEY=$CLERK_SECRET_KEY
ARG CLERK_SIGNING_SECRET
ENV CLERK_SIGNING_SECRET=$CLERK_SIGNING_SECRET
ARG ENABLE_INSTRUMENTATION_HOOK
ENV ENABLE_INSTRUMENTATION_HOOK=$ENABLE_INSTRUMENTATION_HOOK
ENV PNPM_HOME="/pnpm"
@@ -39,8 +16,6 @@ ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
ARG NODE_VERSION=20
RUN apt update \
&& apt install -y curl \
&& curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n \
@@ -72,12 +47,19 @@ WORKDIR /app/apps/dashboard
RUN pnpm install --frozen-lockfile --ignore-scripts
WORKDIR /app
COPY apps apps
COPY apps/dashboard apps/dashboard
COPY packages packages
COPY tooling tooling
RUN pnpm db:codegen
WORKDIR /app/apps/dashboard
# Will be replaced on runtime
ENV NEXT_PUBLIC_DASHBOARD_URL="__NEXT_PUBLIC_DASHBOARD_URL__"
ENV NEXT_PUBLIC_API_URL="__NEXT_PUBLIC_API_URL__"
# Check entrypoint for this little fellow
ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY="pk_test_eW9sby5jb20k"
RUN pnpm run build
# PROD
@@ -116,4 +98,7 @@ WORKDIR /app/apps/dashboard
EXPOSE 3000
CMD ["pnpm", "start"]
# CMD ["pnpm", "start"]
COPY --from=build /app/apps/dashboard/entrypoint.sh /usr/bin/
RUN chmod +x /usr/bin/entrypoint.sh
ENTRYPOINT ["entrypoint.sh", "pnpm", "start"]

32
apps/dashboard/entrypoint.sh Normal file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
set -e
echo "> Replace env variable placeholders with runtime values..."
# Define an array of environment variables to check
variables_to_replace=(
"NEXT_PUBLIC_DASHBOARD_URL"
"NEXT_PUBLIC_API_URL"
"NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"
)
# Replace env variable placeholders with real values
for key in "${variables_to_replace[@]}"; do
value=$(printenv $key)
if [ ! -z "$value" ]; then
echo " - Searching for $key with value $value..."
# Use a custom placeholder for 'NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY' or use the actual key otherwise
if [ "$key" = "NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY" ]; then
placeholder="pk_test_eW9sby5jb20k"
else
placeholder="__${key}__"
fi
# Run the replacement
find /app/apps/dashboard/.next/ -type f \( -name "*.js" -o -name "*.html" \) -exec sed -i "s|$placeholder|$value|g" {} \;
else
echo " - Skipping $key as it has no value set."
fi
done
# Execute the container's main process (CMD in Dockerfile)
exec "$@"

2
apps/dashboard/next.config.mjs
View File

@@ -30,7 +30,7 @@ const config = {
experimental: {
// Avoid "Critical dependency: the request of a dependency is an expression"
serverComponentsExternalPackages: ['bullmq', 'ioredis'],
instrumentationHook: true,
instrumentationHook: !!process.env.ENABLE_INSTRUMENTATION_HOOK,
},
/**
* If you are using `appDir` then you must comment the below `i18n` config out.

6
apps/dashboard/package.json
View File

@@ -15,7 +15,7 @@
"dependencies": {
"@baselime/node-opentelemetry": "^0.5.8",
"@clerk/nextjs": "^5.0.12",
"@clickhouse/client": "^0.2.9",
"@clickhouse/client": "^1.2.0",
"@hookform/resolvers": "^3.3.4",
"@openpanel/common": "workspace:^",
"@openpanel/constants": "workspace:^",
@@ -24,6 +24,7 @@
"@openpanel/queue": "workspace:^",
"@openpanel/sdk-info": "workspace:^",
"@openpanel/validation": "workspace:^",
"@prisma/nextjs-monorepo-workaround-plugin": "^5.12.1",
"@radix-ui/react-accordion": "^1.1.2",
"@radix-ui/react-alert-dialog": "^1.0.5",
"@radix-ui/react-aspect-ratio": "^1.0.3",
@@ -112,7 +113,6 @@
"@openpanel/prettier-config": "workspace:*",
"@openpanel/trpc": "workspace:*",
"@openpanel/tsconfig": "workspace:*",
"@prisma/nextjs-monorepo-workaround-plugin": "^5.12.1",
"@types/bcrypt": "^5.0.2",
"@types/lodash.debounce": "^4.0.9",
"@types/lodash.throttle": "^4.1.9",
@@ -145,4 +145,4 @@
]
},
"prettier": "@openpanel/prettier-config"
}
}

19
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-list/event-listener.tsx
View File

@@ -1,12 +1,12 @@
'use client';
import { useState } from 'react';
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from '@/components/ui/tooltip';
import { useAppParams } from '@/hooks/useAppParams';
import { useDebounceVal } from '@/hooks/useDebounceVal';
import useWS from '@/hooks/useWS';
import { cn } from '@/utils/cn';
import dynamic from 'next/dynamic';
@@ -22,11 +22,13 @@ const AnimatedNumbers = dynamic(() => import('react-animated-numbers'), {
export default function EventListener() {
const router = useRouter();
const { projectId } = useAppParams();
const [counter, setCounter] = useState(0);
const counter = useDebounceVal(0, 1000, {
maxWait: 5000,
});
useWS<IServiceEventMinimal>(`/live/events/${projectId}`, (event) => {
if (event?.name) {
setCounter((prev) => prev + 1);
counter.set((prev) => prev + 1);
}
});
@@ -35,7 +37,7 @@ export default function EventListener() {
<TooltipTrigger asChild>
<button
onClick={() => {
setCounter(0);
counter.set(0);
router.refresh();
}}
className="flex h-8 items-center gap-2 rounded border border-border bg-card px-3 text-sm font-medium leading-none"
@@ -52,7 +54,7 @@ export default function EventListener() {
)}
></div>
</div>
{counter === 0 ? (
{counter.debounced === 0 ? (
'Listening'
) : (
<>
@@ -61,11 +63,10 @@ export default function EventListener() {
transitions={(index) => ({
type: 'spring',
duration: index + 0.3,
damping: 10,
stiffness: 200,
})}
animateToNumber={counter}
animateToNumber={counter.debounced}
locale="en"
/>
new events
@@ -74,7 +75,9 @@ export default function EventListener() {
</button>
</TooltipTrigger>
<TooltipContent side="bottom">
{counter === 0 ? 'Listening to new events' : 'Click to refresh'}
{counter.debounced === 0
? 'Listening to new events'
: 'Click to refresh'}
</TooltipContent>
</Tooltip>
);

2
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx
View File

@@ -5,7 +5,7 @@ import { getEvents } from '@openpanel/db';
import LiveEvents from './live-events';
type Props = {
projectId?: string;
projectId: string;
limit?: number;
};
const RealtimeLiveEventsServer = async ({ projectId, limit = 30 }: Props) => {

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/live-events.tsx
View File

@@ -12,14 +12,14 @@ import type {
type Props = {
events: (IServiceEventMinimal | IServiceCreateEventPayload)[];
projectId?: string;
projectId: string;
limit: number;
};
const RealtimeLiveEvents = ({ events, projectId, limit }: Props) => {
const [state, setState] = useState(events ?? []);
useWS<IServiceEventMinimal | IServiceCreateEventPayload>(
projectId ? `/live/events/${projectId}` : '/live/events',
`/live/events/${projectId}`,
(event) => {
setState((p) => [event, ...p].slice(0, limit));
}

21
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-reloader.tsx
View File

@@ -12,12 +12,21 @@ const RealtimeReloader = ({ projectId }: Props) => {
const client = useQueryClient();
const router = useRouter();
useWS<number>(`/live/visitors/${projectId}`, (value) => {
router.refresh();
client.refetchQueries({
type: 'active',
});
});
useWS<number>(
`/live/events/${projectId}`,
() => {
router.refresh();
client.refetchQueries({
type: 'active',
});
},
{
debounce: {
maxWait: 15000,
delay: 15000,
},
}
);
return null;
};

2
apps/dashboard/src/app/api/clerk/webhook/route.ts
View File

@@ -3,6 +3,8 @@ import { pathOr } from 'ramda';
import { AccessLevel, db } from '@openpanel/db';
export const dynamic = 'force-dynamic';
export async function POST(request: Request) {
const payload: WebhookEvent = await request.json();

3
apps/dashboard/src/app/providers.tsx
View File

@@ -63,7 +63,8 @@ function AllProviders({ children }: { children: React.ReactNode }) {
disableTransitionOnChange
>
<OpenpanelProvider
clientId="301c6dc1-424c-4bc3-9886-a8beab09b615"
url="https://op.coderax.se/api"
clientId="d32780cb-1c60-4a1b-bb5a-ffc11973255e"
profileId={userId || undefined}
trackScreenViews
trackOutgoingLinks

30
apps/dashboard/src/hooks/useDebounceVal.ts Normal file
View File

@@ -0,0 +1,30 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import debounce from 'lodash.debounce';
interface DebouncedState<T> {
value: T;
debounced: T;
set: React.Dispatch<React.SetStateAction<T>>;
}
export function useDebounceVal<T>(
initialValue: T,
delay = 500,
options?: Parameters<typeof debounce>[2]
): DebouncedState<T> {
const [value, setValue] = useState<T>(initialValue);
const [debouncedValue, _setDebouncedValue] = useState<T>(initialValue);
const setDebouncedValue = useMemo(
() => debounce(_setDebouncedValue, delay, options),
[]
);
useEffect(() => {
setDebouncedValue(value);
}, [value]);
return {
value,
debounced: debouncedValue,
set: setValue,
};
}

22
apps/dashboard/src/hooks/useWS.ts
View File

@@ -2,11 +2,22 @@
import { use, useEffect, useMemo, useState } from 'react';
import { useAuth } from '@clerk/nextjs';
import debounce from 'lodash.debounce';
import useWebSocket from 'react-use-websocket';
import { getSuperJson } from '@openpanel/common';
export default function useWS<T>(path: string, onMessage: (event: T) => void) {
type UseWSOptions = {
debounce?: {
delay: number;
} & Parameters<typeof debounce>[2];
};
export default function useWS<T>(
path: string,
onMessage: (event: T) => void,
options?: UseWSOptions
) {
const auth = useAuth();
const ws = String(process.env.NEXT_PUBLIC_API_URL)
.replace(/^https/, 'wss')
@@ -18,6 +29,13 @@ export default function useWS<T>(path: string, onMessage: (event: T) => void) {
[baseUrl, token]
);
const debouncedOnMessage = useMemo(() => {
if (options?.debounce) {
return debounce(onMessage, options.debounce.delay, options.debounce);
}
return onMessage;
}, [options?.debounce?.delay]);
useEffect(() => {
if (auth.isSignedIn) {
auth.getToken().then(setToken);
@@ -35,7 +53,7 @@ export default function useWS<T>(path: string, onMessage: (event: T) => void) {
try {
const data = getSuperJson<T>(event.data);
if (data) {
onMessage(data);
debouncedOnMessage(data);
}
} catch (error) {
console.error('Error parsing message', error);

33
apps/public/Dockerfile
View File

@@ -1,43 +1,16 @@
FROM --platform=linux/amd64 node:20-slim AS base
ARG NODE_VERSION=20
ARG NEXT_PUBLIC_DASHBOARD_URL
ENV NEXT_PUBLIC_DASHBOARD_URL=$NEXT_PUBLIC_DASHBOARD_URL
ARG NEXT_PUBLIC_API_URL
ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
FROM --platform=linux/amd64 node:${NODE_VERSION}-slim AS base
ARG DATABASE_URL
ENV DATABASE_URL=$DATABASE_URL
ARG CLICKHOUSE_DB
ENV CLICKHOUSE_DB=$CLICKHOUSE_DB
ARG CLICKHOUSE_PASSWORD
ENV CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
ARG CLICKHOUSE_URL
ENV CLICKHOUSE_URL=$CLICKHOUSE_URL
ARG CLICKHOUSE_USER
ENV CLICKHOUSE_USER=$CLICKHOUSE_USER
ARG REDIS_URL
ENV REDIS_URL=$REDIS_URL
ARG NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
ARG CLERK_SECRET_KEY
ENV CLERK_SECRET_KEY=$CLERK_SECRET_KEY
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
ARG NODE_VERSION=20
RUN apt update \
&& apt install -y curl \
&& curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n \
@@ -66,7 +39,7 @@ WORKDIR /app/apps/public
RUN pnpm install --frozen-lockfile --ignore-scripts
WORKDIR /app
COPY apps apps
COPY apps/public apps/public
COPY packages packages
COPY tooling tooling
RUN pnpm db:codegen

24
apps/worker/Dockerfile
View File

@@ -1,33 +1,16 @@
# Dockerfile that builds the web app only
ARG NODE_VERSION=20
FROM --platform=linux/amd64 node:20-slim AS base
FROM --platform=linux/amd64 node:${NODE_VERSION}-slim AS base
ARG DATABASE_URL
ENV DATABASE_URL=$DATABASE_URL
ARG CLICKHOUSE_DB
ENV CLICKHOUSE_DB=$CLICKHOUSE_DB
ARG CLICKHOUSE_PASSWORD
ENV CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
ARG CLICKHOUSE_URL
ENV CLICKHOUSE_URL=$CLICKHOUSE_URL
ARG CLICKHOUSE_USER
ENV CLICKHOUSE_USER=$CLICKHOUSE_USER
ARG REDIS_URL
ENV REDIS_URL=$REDIS_URL
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
RUN corepack enable
ARG NODE_VERSION=20
RUN apt update \
&& apt install -y curl \
&& curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n \
@@ -49,6 +32,7 @@ COPY packages/common/package.json packages/common/package.json
COPY packages/constants/package.json packages/constants/package.json
COPY packages/validation/package.json packages/validation/package.json
COPY packages/sdks/sdk/package.json packages/sdks/sdk/package.json
COPY patches patches
# BUILD
FROM base AS build
@@ -57,7 +41,7 @@ WORKDIR /app/apps/worker
RUN pnpm install --frozen-lockfile --ignore-scripts
WORKDIR /app
COPY apps apps
COPY apps/worker apps/worker
COPY packages packages
COPY tooling tooling
RUN pnpm db:codegen

7
apps/worker/package.json
View File

@@ -12,16 +12,17 @@
},
"dependencies": {
"@baselime/pino-transport": "^0.1.5",
"@bull-board/api": "^5.13.0",
"@bull-board/express": "^5.13.0",
"@bull-board/api": "^5.21.0",
"@bull-board/express": "^5.21.0",
"@openpanel/common": "workspace:*",
"@openpanel/db": "workspace:*",
"@openpanel/logger": "workspace:*",
"@openpanel/queue": "workspace:*",
"@openpanel/redis": "workspace:*",
"bullmq": "^5.1.1",
"bullmq": "^5.8.7",
"express": "^4.18.2",
"pino": "^8.17.2",
"pino-pretty": "^10.3.1",
"ramda": "^0.29.1",
"sqlstring": "^2.3.3",
"ua-parser-js": "^1.0.37",

139
apps/worker/scripts/debug.ts Normal file
View File

@@ -0,0 +1,139 @@
import { escape } from 'sqlstring';
import type { IClickhouseEvent } from '@openpanel/db';
import { chQuery, eventBuffer } from '@openpanel/db';
import { sessionsQueue } from '@openpanel/queue/src/queues';
import { redis } from '@openpanel/redis';
async function debugStalledEvents() {
const keys = await redis.keys('bull:sessions:sessionEnd*');
const delayedZRange = await redis.zrange(
'bull:sessions:delayed',
0,
-1,
'WITHSCORES'
);
const delayedValues = delayedZRange.reduce(
(acc, item, index, array) => {
if (index % 2 === 0) {
acc[item] = Number(array[index + 1]) / 0x1000;
}
return acc;
},
[] as Record<string, number>
);
const opKeys = await redis.keys('op:*');
const stalledEvents = await redis.lrange('op:buffer:events:stalled', 0, -1);
// keys.forEach((key) => {
// console.log(key);
// });
// console.log('--------------------');
const queue = await eventBuffer.getQueue(-1);
queue
.filter((item) => item.event.name === 'screen_view')
.forEach((item) => {
const date = new Date(item.event.created_at.replace(' ', 'T') + 'Z');
const match = keys.find((key) => {
return item.event.device_id && key.includes(item.event.device_id);
});
if (match) {
// console.log(
// date.toISOString(),
// item.event.name,
// item.event.device,
// item.event.session_id,
// item.event.profile_id,
// item.event.device_id,
// match
// );
} else {
console.log(
'NO MATCH FOUND!',
date.toISOString(),
item.event.name,
'[SID]',
item.event.session_id,
'[PID]',
item.event.profile_id,
'[DID]',
item.event.device_id
);
console.log(item.event);
console.log('');
// console.log('Not in queue!');
// log§
}
});
if (stalledEvents.length > 0) {
const res = await chQuery(
`SELECT * FROM events WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`
);
stalledEvents.forEach((item) => {
const event = JSON.parse(item) as IClickhouseEvent;
const date = new Date(event.created_at.replace(' ', 'T') + 'Z');
console.log(
'STALLED!',
date.toISOString(),
event.name,
'[IN_DB]',
res.find((item) => item.id === event.id) ? 'YES' : 'NO',
'[ID]',
event.id,
'[SID]',
event.session_id,
'[PID]',
event.profile_id,
'[DID]',
event.device_id
);
// console.log(event);
});
}
console.log('OP Keys', opKeys);
console.log('Queue', queue.length);
console.log('Session Ends', keys.length);
console.log('Stalled Events', stalledEvents.length);
// keys.forEach((key) => {
// if (key.includes('e1b233e69bcd2132ec7bf343004d4b01')) {
// console.log(key);
// }
// });
const delayedJobs = await sessionsQueue.getDelayed();
console.log('delayedJobs', delayedJobs.length);
delayedJobs.sort((a, b) => a.timestamp + a.delay - (b.timestamp + b.delay));
let delayedJobsCount = 0;
delayedJobs.forEach((job) => {
const date = new Date(delayedValues[job.id]);
// if date is in the past
// if (date.getTime() - 1000 * 60 * 5 < Date.now()) {
if (date.getTime() < Date.now()) {
delayedJobsCount++;
console.log(
date.toLocaleString('sv-SE'),
'https://op.coderax.se/worker/queue/sessions/' +
encodeURIComponent(job.id)
);
}
});
console.log('delayedJobsCount', delayedJobsCount);
}
async function main() {
if (process.argv[2] === 'stalled') {
await debugStalledEvents();
}
process.exit(0);
}
main();

121
apps/worker/src/index.ts
View File

@@ -6,28 +6,42 @@ import { Worker } from 'bullmq';
import express from 'express';
import { connection, eventsQueue } from '@openpanel/queue';
import { cronQueue } from '@openpanel/queue/src/queues';
import { cronQueue, sessionsQueue } from '@openpanel/queue/src/queues';
import { cronJob } from './jobs/cron';
import { eventsJob } from './jobs/events';
import { sessionsJob } from './jobs/sessions';
const PORT = parseInt(process.env.WORKER_PORT || '3000', 10);
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath('/');
serverAdapter.setBasePath(process.env.SELF_HOSTED ? '/worker' : '/');
const app = express();
const workerOptions: WorkerOptions = {
connection,
connection: {
...connection,
enableReadyCheck: false,
maxRetriesPerRequest: null,
enableOfflineQueue: true,
},
concurrency: parseInt(process.env.CONCURRENCY || '1', 10),
};
async function start() {
new Worker(eventsQueue.name, eventsJob, workerOptions);
new Worker(cronQueue.name, cronJob, workerOptions);
const eventsWorker = new Worker(eventsQueue.name, eventsJob, workerOptions);
const sessionsWorker = new Worker(
sessionsQueue.name,
sessionsJob,
workerOptions
);
const cronWorker = new Worker(cronQueue.name, cronJob, workerOptions);
createBullBoard({
queues: [new BullMQAdapter(eventsQueue), new BullMQAdapter(cronQueue)],
queues: [
new BullMQAdapter(eventsQueue),
new BullMQAdapter(sessionsQueue),
new BullMQAdapter(cronQueue),
],
serverAdapter: serverAdapter,
});
@@ -37,9 +51,61 @@ async function start() {
console.log(`For the UI, open http://localhost:${PORT}/`);
});
const repeatableJobs = await cronQueue.getRepeatableJobs();
function workerLogger(worker: string, type: string, err?: Error) {
console.log(`Worker ${worker} -> ${type}`);
if (err) {
console.error(err);
}
}
console.log(repeatableJobs);
const workers = [sessionsWorker, eventsWorker, cronWorker];
workers.forEach((worker) => {
worker.on('error', (err) => {
workerLogger(worker.name, 'error', err);
});
worker.on('closed', () => {
workerLogger(worker.name, 'closed');
});
worker.on('ready', () => {
workerLogger(worker.name, 'ready');
});
});
async function exitHandler(evtOrExitCodeOrError: number | string | Error) {
try {
await eventsWorker.close();
await sessionsWorker.close();
await cronWorker.close();
} catch (e) {
console.error('EXIT HANDLER ERROR', e);
}
process.exit(isNaN(+evtOrExitCodeOrError) ? 1 : +evtOrExitCodeOrError);
}
[
'beforeExit',
'uncaughtException',
'unhandledRejection',
'SIGHUP',
'SIGINT',
'SIGQUIT',
'SIGILL',
'SIGTRAP',
'SIGABRT',
'SIGBUS',
'SIGFPE',
'SIGUSR1',
'SIGSEGV',
'SIGUSR2',
'SIGTERM',
].forEach((evt) =>
process.on(evt, (evt) => {
exitHandler(evt);
})
);
await cronQueue.add(
'salt',
@@ -55,6 +121,43 @@ async function start() {
},
}
);
await cronQueue.add(
'flush',
{
type: 'flushEvents',
payload: undefined,
},
{
jobId: 'flushEvents',
repeat: {
every: process.env.BATCH_INTERVAL
? parseInt(process.env.BATCH_INTERVAL, 10)
: 1000 * 10,
},
}
);
await cronQueue.add(
'flush',
{
type: 'flushProfiles',
payload: undefined,
},
{
jobId: 'flushProfiles',
repeat: {
every: process.env.BATCH_INTERVAL
? parseInt(process.env.BATCH_INTERVAL, 10)
: 1000 * 10,
},
}
);
const repeatableJobs = await cronQueue.getRepeatableJobs();
console.log('Repeatable jobs:');
console.log(repeatableJobs);
}
start();

4
apps/worker/src/jobs/cron.salt.ts
View File

@@ -2,7 +2,7 @@ import { generateSalt } from '@openpanel/common';
import { db, getCurrentSalt } from '@openpanel/db';
export async function salt() {
const oldSalt = await getCurrentSalt();
const oldSalt = await getCurrentSalt().catch(() => null);
const newSalt = await db.salt.create({
data: {
salt: generateSalt(),
@@ -13,7 +13,7 @@ export async function salt() {
await db.salt.deleteMany({
where: {
salt: {
notIn: [newSalt.salt, oldSalt],
notIn: oldSalt ? [newSalt.salt, oldSalt] : [newSalt.salt],
},
},
});

7
apps/worker/src/jobs/cron.ts
View File

@@ -1,5 +1,6 @@
import type { Job } from 'bullmq';
import { eventBuffer, profileBuffer } from '@openpanel/db';
import type { CronQueuePayload } from '@openpanel/queue/src/queues';
import { salt } from './cron.salt';
@@ -9,5 +10,11 @@ export async function cronJob(job: Job<CronQueuePayload>) {
case 'salt': {
return await salt();
}
case 'flushEvents': {
return await eventBuffer.flush();
}
case 'flushProfiles': {
return await profileBuffer.flush();
}
}
}

17
apps/worker/src/jobs/events.create-session-end.ts
View File

@@ -1,23 +1,26 @@
import type { Job } from 'bullmq';
import { getTime } from '@openpanel/common';
import { createEvent, getEvents } from '@openpanel/db';
import { createEvent, eventBuffer, getEvents } from '@openpanel/db';
import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue/src/queues';
export async function createSessionEnd(
job: Job<EventsQueuePayloadCreateSessionEnd>
) {
const payload = job.data.payload;
const eventsInBuffer = await eventBuffer.findMany(
(item) => item.event.session_id === payload.sessionId
);
const sql = `
SELECT * FROM events
WHERE
device_id = '${payload.deviceId}'
session_id = '${payload.sessionId}'
AND created_at >= (
SELECT created_at
FROM events
WHERE
device_id = '${payload.deviceId}'
session_id = '${payload.sessionId}'
AND name = 'session_start'
ORDER BY created_at DESC
LIMIT 1
@@ -25,7 +28,11 @@ export async function createSessionEnd(
ORDER BY created_at DESC
`;
job.log(sql);
const events = await getEvents(sql);
const eventsInDb = await getEvents(sql);
// sort last inserted first
const events = [...eventsInBuffer, ...eventsInDb].sort(
(a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
);
events.map((event, index) => {
job.log(
@@ -64,7 +71,7 @@ export async function createSessionEnd(
},
name: 'session_end',
duration: sessionDuration,
path: lastEvent.path,
path: screenViews[0]?.path ?? '',
createdAt: new Date(getTime(lastEvent?.createdAt) + 100),
});
}

314
apps/worker/src/jobs/events.incoming-event.ts
View File

@@ -1,20 +1,30 @@
import { logger } from '@/utils/logger';
import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
import { isUserAgentSet, parseUserAgent } from '@/utils/parse-user-agent';
import { parseUserAgent } from '@/utils/parse-user-agent';
import { isSameDomain, parsePath } from '@/utils/url';
import type { Job, JobsOptions } from 'bullmq';
import type { Job } from 'bullmq';
import { omit } from 'ramda';
import { escape } from 'sqlstring';
import { v4 as uuid } from 'uuid';
import { getTime, toISOString } from '@openpanel/common';
import type { IServiceCreateEventPayload } from '@openpanel/db';
import { createEvent, getEvents } from '@openpanel/db';
import { createEvent } from '@openpanel/db';
import { getLastScreenViewFromProfileId } from '@openpanel/db/src/services/event.service';
import { findJobByPrefix } from '@openpanel/queue';
import { eventsQueue } from '@openpanel/queue/src/queues';
import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue/src/queues';
import { eventsQueue, sessionsQueue } from '@openpanel/queue/src/queues';
import type {
EventsQueuePayloadCreateSessionEnd,
EventsQueuePayloadIncomingEvent,
} from '@openpanel/queue/src/queues';
import { redis } from '@openpanel/redis';
function noDateInFuture(eventDate: Date): Date {
if (eventDate > new Date()) {
return new Date();
} else {
return eventDate;
}
}
const GLOBAL_PROPERTIES = ['__path', '__referrer'];
const SESSION_TIMEOUT = 1000 * 60 * 30;
const SESSION_END_TIMEOUT = SESSION_TIMEOUT + 1000;
@@ -27,12 +37,8 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
projectId,
currentDeviceId,
previousDeviceId,
// TODO: Remove after 2024-09-26
currentDeviceIdDeprecated,
previousDeviceIdDeprecated,
priority,
} = job.data.payload;
let deviceId: string | null = null;
const properties = body.properties ?? {};
const getProperty = (name: string): string | undefined => {
// replace thing is just for older sdks when we didn't have `__`
@@ -44,22 +50,22 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
| undefined) ?? undefined
);
};
const { ua } = headers;
const profileId = body.profileId ?? '';
const createdAt = new Date(body.timestamp);
const profileId = body.profileId ? String(body.profileId) : '';
const createdAt = noDateInFuture(new Date(body.timestamp));
const url = getProperty('__path');
const { path, hash, query, origin } = parsePath(url);
const referrer = isSameDomain(getProperty('__referrer'), url)
? null
: parseReferrer(getProperty('__referrer'));
const utmReferrer = getReferrerWithQuery(query);
const uaInfo = ua ? parseUserAgent(ua) : null;
const isServerEvent = ua ? !isUserAgentSet(ua) : true;
const uaInfo = parseUserAgent(headers.ua);
if (isServerEvent) {
const [event] = await getEvents(
`SELECT * FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`
);
if (uaInfo.isServer) {
const event = await getLastScreenViewFromProfileId({
profileId,
projectId,
});
const payload: Omit<IServiceCreateEventPayload, 'id'> = {
name: body.name,
@@ -67,7 +73,10 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
sessionId: event?.sessionId || '',
profileId,
projectId,
properties: Object.assign({}, omit(GLOBAL_PROPERTIES, properties)),
properties: {
...omit(GLOBAL_PROPERTIES, properties),
user_agent: headers.ua,
},
createdAt,
country: event?.country || geo.country || '',
city: event?.city || geo.city || '',
@@ -78,7 +87,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
osVersion: event?.osVersion ?? '',
browser: event?.browser ?? '',
browserVersion: event?.browserVersion ?? '',
device: event?.device ?? '',
device: event?.device ?? uaInfo.device ?? '',
brand: event?.brand ?? '',
model: event?.model ?? '',
duration: 0,
@@ -94,82 +103,50 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
return createEvent(payload);
}
const [sessionEndKeys, eventsKeys] = await Promise.all([
redis.keys(`bull:events:sessionEnd:${projectId}:*`),
redis.keys(`bull:events:event:${projectId}:*`),
]);
const sessionEnd = await getSessionEndWithPriority(priority)({
projectId,
currentDeviceId,
previousDeviceId,
});
const sessionEndJobCurrentDeviceId = await findJobByPrefix(
eventsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${currentDeviceId}:`
);
const sessionEndJobPreviousDeviceId = await findJobByPrefix(
eventsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${previousDeviceId}:`
);
// TODO: Remove after 2024-09-26
const sessionEndJobCurrentDeviceIdDeprecated = await findJobByPrefix(
eventsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${currentDeviceIdDeprecated}:`
);
const sessionEndJobPreviousDeviceIdDeprecated = await findJobByPrefix(
eventsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${previousDeviceIdDeprecated}:`
);
const sessionEndPayload = (sessionEnd?.job?.data
?.payload as EventsQueuePayloadCreateSessionEnd['payload']) || {
sessionId: uuid(),
deviceId: currentDeviceId,
profileId,
};
let createSessionStart = false;
const sessionEndJobId =
sessionEnd?.job.id ??
`sessionEnd:${projectId}:${sessionEndPayload.deviceId}:${Date.now()}`;
if (sessionEndJobCurrentDeviceId) {
deviceId = currentDeviceId;
sessionEndJobCurrentDeviceId.changeDelay(SESSION_END_TIMEOUT);
} else if (sessionEndJobPreviousDeviceId) {
deviceId = previousDeviceId;
sessionEndJobPreviousDeviceId.changeDelay(SESSION_END_TIMEOUT);
} else if (sessionEndJobCurrentDeviceIdDeprecated) {
deviceId = currentDeviceIdDeprecated;
sessionEndJobCurrentDeviceIdDeprecated.changeDelay(SESSION_END_TIMEOUT);
} else if (sessionEndJobPreviousDeviceIdDeprecated) {
deviceId = previousDeviceIdDeprecated;
sessionEndJobPreviousDeviceIdDeprecated.changeDelay(SESSION_END_TIMEOUT);
if (sessionEnd) {
// If for some reason we have a session end job that is not a createSessionEnd job
if (sessionEnd.job.data.type !== 'createSessionEnd') {
throw new Error('Invalid session end job');
}
await sessionEnd.job.changeDelay(SESSION_TIMEOUT);
} else {
deviceId = currentDeviceId;
createSessionStart = true;
// Queue session end
eventsQueue.add(
'event',
await sessionsQueue.add(
'session',
{
type: 'createSessionEnd',
payload: {
deviceId,
},
payload: sessionEndPayload,
},
{
delay: SESSION_END_TIMEOUT,
jobId: `sessionEnd:${projectId}:${deviceId}:${Date.now()}`,
jobId: sessionEndJobId,
}
);
}
const prevEventJob = await findJobByPrefix(
eventsQueue,
eventsKeys,
`event:${projectId}:${deviceId}:`
);
const [sessionStartEvent] = await getEvents(
`SELECT * FROM events WHERE name = 'session_start' AND device_id = ${escape(deviceId)} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`
);
const payload: Omit<IServiceCreateEventPayload, 'id'> = {
name: body.name,
deviceId,
deviceId: sessionEndPayload.deviceId,
sessionId: sessionEndPayload.sessionId,
profileId,
projectId,
sessionId: createSessionStart ? uuid() : sessionStartEvent?.sessionId ?? '',
properties: Object.assign({}, omit(GLOBAL_PROPERTIES, properties), {
__hash: hash,
__query: query,
@@ -189,7 +166,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
model: uaInfo?.model ?? '',
duration: 0,
path: path,
origin: origin || sessionStartEvent?.origin || '',
origin: origin,
referrer: referrer?.url,
referrerName: referrer?.name || utmReferrer?.name || '',
referrerType: referrer?.type || utmReferrer?.type || '',
@@ -197,76 +174,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
meta: undefined,
};
const isDelayed = prevEventJob ? await prevEventJob?.isDelayed() : false;
if (isDelayed && prevEventJob && prevEventJob.data.type === 'createEvent') {
const prevEvent = prevEventJob.data.payload;
const duration = getTime(payload.createdAt) - getTime(prevEvent.createdAt);
job.log(`prevEvent ${JSON.stringify(prevEvent, null, 2)}`);
// Set path from prev screen_view event if current event is not a screen_view
if (payload.name != 'screen_view') {
payload.path = prevEvent.path;
}
if (payload.name === 'screen_view') {
if (duration < 0) {
logger.info({ prevEvent, payload }, 'Duration is negative');
} else {
try {
// Skip update duration if it's wrong
// Seems like request is not in right order
await prevEventJob.updateData({
type: 'createEvent',
payload: {
...prevEvent,
duration,
},
});
} catch (error) {
logger.error(
{
error,
prevEventJobStatus: await prevEventJob
.getState()
.catch(() => 'unknown'),
},
`Failed update delayed job`
);
}
}
try {
await prevEventJob.promote();
} catch (error) {
logger.error(
{
error,
prevEventJobStatus: await prevEventJob
.getState()
.catch(() => 'unknown'),
prevEvent,
currEvent: payload,
},
`Failed to promote job`
);
}
}
} else if (payload.name !== 'screen_view') {
job.log(
`no previous job ${JSON.stringify(
{
prevEventJob,
payload,
},
null,
2
)}`
);
}
if (createSessionStart) {
// We do not need to queue session_start
if (!sessionEnd) {
await createEvent({
...payload,
name: 'session_start',
@@ -275,40 +183,78 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
});
}
const options: JobsOptions = {};
if (payload.name === 'screen_view') {
options.delay = SESSION_TIMEOUT;
options.jobId = `event:${projectId}:${deviceId}:${Date.now()}`;
return createEvent(payload);
}
function getSessionEndWithPriority(
priority: boolean,
count = 0
): typeof getSessionEnd {
return async (args) => {
const res = await getSessionEnd(args);
if (count > 5) {
throw new Error('Failed to get session end');
}
// if we get simultaneous requests we want to avoid race conditions with getting the session end
// one of the events will get priority and the other will wait for the first to finish
if (res === null && priority === false) {
await new Promise((resolve) => setTimeout(resolve, 50));
return getSessionEndWithPriority(priority, count + 1)(args);
}
return res;
};
}
async function getSessionEnd({
projectId,
currentDeviceId,
previousDeviceId,
}: {
projectId: string;
currentDeviceId: string;
previousDeviceId: string;
}) {
const sessionEndKeys = await redis.keys(`*:sessionEnd:${projectId}:*`);
const sessionEndJobCurrentDeviceId = await findJobByPrefix(
sessionsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${currentDeviceId}:`
);
if (sessionEndJobCurrentDeviceId) {
return { deviceId: currentDeviceId, job: sessionEndJobCurrentDeviceId };
}
job.log(
`event is queued ${JSON.stringify(
{
ua,
uaInfo,
referrer,
profileId,
projectId,
deviceId,
geo,
sessionStartEvent,
path,
payload,
},
null,
2
)}`
const sessionEndJobCurrentDeviceId2 = await findJobByPrefix(
eventsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${currentDeviceId}:`
);
if (sessionEndJobCurrentDeviceId2) {
return { deviceId: currentDeviceId, job: sessionEndJobCurrentDeviceId2 };
}
// Queue event instead of creating it,
// since we want to update duration if we get more events in the same session
// The event will only be delayed if it's a screen_view event
return eventsQueue.add(
'event',
{
type: 'createEvent',
payload,
},
options
const sessionEndJobPreviousDeviceId = await findJobByPrefix(
sessionsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${previousDeviceId}:`
);
if (sessionEndJobPreviousDeviceId) {
return { deviceId: previousDeviceId, job: sessionEndJobPreviousDeviceId };
}
const sessionEndJobPreviousDeviceId2 = await findJobByPrefix(
eventsQueue,
sessionEndKeys,
`sessionEnd:${projectId}:${previousDeviceId}:`
);
if (sessionEndJobPreviousDeviceId2) {
return { deviceId: previousDeviceId, job: sessionEndJobPreviousDeviceId2 };
}
// Create session
return null;
}

18
apps/worker/src/jobs/events.ts
View File

@@ -1,7 +1,7 @@
import type { Job } from 'bullmq';
import { escape } from 'sqlstring';
import { chQuery, createEvent, db } from '@openpanel/db';
import { chQuery, db } from '@openpanel/db';
import type {
EventsQueuePayload,
EventsQueuePayloadCreateSessionEnd,
@@ -16,22 +16,6 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {
case 'incomingEvent': {
return await incomingEvent(job as Job<EventsQueuePayloadIncomingEvent>);
}
case 'createEvent': {
if (job.attemptsStarted > 1 && job.data.payload.duration < 0) {
job.data.payload.duration = 0;
}
const createdEvent = await createEvent(job.data.payload);
try {
await updateEventsCount(job.data.payload.projectId);
} catch (e) {
if (e instanceof Error) {
job.log(`Failed to update events count: ${e.message}`);
} else {
job.log(`Failed to update events count: Unknown issue`);
}
}
return createdEvent;
}
case 'createSessionEnd': {
return await createSessionEnd(
job as Job<EventsQueuePayloadCreateSessionEnd>

9
apps/worker/src/jobs/sessions.ts Normal file
View File

@@ -0,0 +1,9 @@
import type { Job } from 'bullmq';
import type { SessionsQueuePayload } from '@openpanel/queue/src/queues';
import { createSessionEnd } from './events.create-session-end';
export async function sessionsJob(job: Job<SessionsQueuePayload>) {
return await createSessionEnd(job);
}

87
apps/worker/src/utils/parse-user-agent.ts
View File

@@ -1,11 +1,16 @@
import { UAParser } from 'ua-parser-js';
export function isUserAgentSet(ua: string) {
return ua !== 'node' && ua !== 'undici' && !!ua;
}
const parsedServerUa = {
isServer: true,
device: 'server',
} as const;
export function parseUserAgent(ua: string) {
export function parseUserAgent(ua?: string | null) {
if (!ua) return parsedServerUa;
const res = new UAParser(ua).getResult();
if (isServer(ua)) return parsedServerUa;
return {
os: res.os.name,
osVersion: res.os.version,
@@ -14,20 +19,86 @@ export function parseUserAgent(ua: string) {
device: res.device.type ?? getDevice(ua),
brand: res.device.vendor,
model: res.device.model,
};
isServer: false,
} as const;
}
const userAgentServerList: string[] = [
// Node.js libraries
'node',
'node-fetch',
'axios',
'request',
'superagent',
'undici',
// Python libraries
'python-requests',
'python-urllib',
// Ruby libraries
'Faraday',
'Ruby',
'http.rb',
// Go libraries
'Go-http-client',
'Go-http-client',
// Java libraries
'Apache-HttpClient',
'okhttp',
'okhowtp',
// PHP libraries
'GuzzleHttp',
'PHP-cURL',
// Other
'Dart',
'RestSharp', // Popular .NET HTTP client library
'HttpClientFactory', // .NET's typed client factory
'Ktor', // A client for Kotlin
'Ning', // Async HTTP client for Java
'grpc-csharp', // gRPC for C#
'Volley', // HTTP library used in Android apps for making network requests
'Spring',
'vert.x',
'grpc-',
];
function isServer(userAgent: string) {
const match = userAgentServerList.some((server) =>
userAgent.includes(server)
);
if (match) {
return true;
}
return !!userAgent.match(/^[^\s]+\/[\d.]+$/);
}
export function getDevice(ua: string) {
const t1 =
const mobile1 =
/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
ua
);
const t2 =
const mobile2 =
/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
ua.slice(0, 4)
);
if (t1 || t2) {
const tablet =
/tablet|ipad|android(?!.*mobile)|xoom|sch-i800|kindle|silk|playbook/i.test(
ua
);
if (mobile1 || mobile2) {
return 'mobile';
}
if (tablet) {
return 'tablet';
}
return 'desktop';
}