batching events
This commit is contained in:
committed by
Carl-Gerhard Lindesvärd
parent
244aa3b0d3
commit
5e225b7ae6
@@ -1,48 +1,16 @@
|
||||
# Dockerfile that builds the web app only
|
||||
|
||||
FROM --platform=linux/amd64 node:20-slim AS base
|
||||
ARG NODE_VERSION=20
|
||||
FROM --platform=linux/amd64 node:${NODE_VERSION}-slim AS base
|
||||
|
||||
ARG DATABASE_URL
|
||||
ENV DATABASE_URL=$DATABASE_URL
|
||||
|
||||
ARG CLICKHOUSE_DB
|
||||
ENV CLICKHOUSE_DB=$CLICKHOUSE_DB
|
||||
|
||||
ARG CLICKHOUSE_PASSWORD
|
||||
ENV CLICKHOUSE_PASSWORD=$CLICKHOUSE_PASSWORD
|
||||
|
||||
ARG CLICKHOUSE_URL
|
||||
ENV CLICKHOUSE_URL=$CLICKHOUSE_URL
|
||||
|
||||
ARG CLICKHOUSE_USER
|
||||
ENV CLICKHOUSE_USER=$CLICKHOUSE_USER
|
||||
|
||||
ARG REDIS_URL
|
||||
ENV REDIS_URL=$REDIS_URL
|
||||
|
||||
ARG NEXT_PUBLIC_DASHBOARD_URL
|
||||
ENV NEXT_PUBLIC_DASHBOARD_URL=$NEXT_PUBLIC_DASHBOARD_URL
|
||||
|
||||
ARG NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
|
||||
ENV NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=$NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY
|
||||
|
||||
ARG CLERK_SECRET_KEY
|
||||
ENV CLERK_SECRET_KEY=$CLERK_SECRET_KEY
|
||||
|
||||
ARG CLERK_PUBLIC_PEM_KEY
|
||||
ENV CLERK_PUBLIC_PEM_KEY=$CLERK_PUBLIC_PEM_KEY
|
||||
|
||||
ARG SEVENTY_SEVEN_API_KEY
|
||||
ENV SEVENTY_SEVEN_API_KEY=$SEVENTY_SEVEN_API_KEY
|
||||
|
||||
ENV PNPM_HOME="/pnpm"
|
||||
|
||||
ENV PATH="$PNPM_HOME:$PATH"
|
||||
|
||||
RUN corepack enable
|
||||
|
||||
ARG NODE_VERSION=20
|
||||
|
||||
RUN apt update \
|
||||
&& apt install -y curl python3 make g++ \
|
||||
&& curl -L https://raw.githubusercontent.com/tj/n/master/bin/n -o n \
|
||||
@@ -73,7 +41,7 @@ WORKDIR /app/apps/api
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
WORKDIR /app
|
||||
COPY apps apps
|
||||
COPY apps/api apps/api
|
||||
COPY packages packages
|
||||
COPY tooling tooling
|
||||
RUN pnpm db:codegen
|
||||
|
||||
@@ -35,6 +35,7 @@
|
||||
"sharp": "^0.33.2",
|
||||
"sqlstring": "^2.3.3",
|
||||
"superjson": "^1.13.3",
|
||||
"svix": "^1.24.0",
|
||||
"ua-parser-js": "^1.0.37",
|
||||
"url-metadata": "^4.1.0",
|
||||
"zod": "^3.22.4"
|
||||
|
||||
189
apps/api/scripts/get-organizations.ts
Normal file
189
apps/api/scripts/get-organizations.ts
Normal file
@@ -0,0 +1,189 @@
|
||||
// import { clerkClient } from '@clerk/fastify';
|
||||
|
||||
import { db } from '@openpanel/db';
|
||||
|
||||
// import { db } from '@openpanel/db';
|
||||
|
||||
// type Fn<T = unknown> = (args: { limit: number; offset: number }) => Promise<{
|
||||
// data: T[];
|
||||
// totalCount: number;
|
||||
// }>;
|
||||
|
||||
// function getAllDataByPagination<T extends Fn>(
|
||||
// cb: T
|
||||
// ): Promise<Awaited<ReturnType<T>>['data']> {
|
||||
// const data: Awaited<ReturnType<T>>['data'] = [];
|
||||
// async function getData(page = 0) {
|
||||
// console.log(`getData with offset ${page * 100}`);
|
||||
// const response = await cb({
|
||||
// limit: 100,
|
||||
// offset: page * 100,
|
||||
// });
|
||||
// if (response.data.length !== 0) {
|
||||
// data.push(...response.data);
|
||||
// await getData(page + 1);
|
||||
// }
|
||||
// await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
// }
|
||||
|
||||
// return getData().then(() => data);
|
||||
// }
|
||||
|
||||
// async function main() {
|
||||
// const organizations = await getAllDataByPagination(
|
||||
// clerkClient.organizations.getOrganizationList.bind(
|
||||
// clerkClient.organizations
|
||||
// )
|
||||
// );
|
||||
// const users = await getAllDataByPagination(
|
||||
// clerkClient.users.getUserList.bind(clerkClient.users)
|
||||
// );
|
||||
|
||||
// console.log(`Found ${organizations.length} organizations`);
|
||||
// console.log(`Found ${users.length} users`);
|
||||
|
||||
// for (const user of users.slice(-10)) {
|
||||
// const email = user.primaryEmailAddress?.emailAddress;
|
||||
// console.log('Check', email);
|
||||
|
||||
// try {
|
||||
// if (email) {
|
||||
// const exists = await db.user.findUnique({
|
||||
// where: {
|
||||
// id: user.id,
|
||||
// },
|
||||
// });
|
||||
|
||||
// if (exists) {
|
||||
// console.log('already exists');
|
||||
// } else {
|
||||
// await db.user.create({
|
||||
// data: {
|
||||
// id: user.id,
|
||||
// email: email,
|
||||
// firstName: user.firstName,
|
||||
// lastName: user.lastName,
|
||||
// },
|
||||
// });
|
||||
// }
|
||||
// } else {
|
||||
// console.log('No email?', user);
|
||||
// }
|
||||
// } catch (e) {
|
||||
// console.log('ERROR');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.dir(user, { depth: null });
|
||||
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// }
|
||||
// }
|
||||
|
||||
// for (const org of organizations.slice(-20)) {
|
||||
// try {
|
||||
// if (org.slug) {
|
||||
// const exists = await db.organization.findUnique({
|
||||
// where: {
|
||||
// id: org.slug,
|
||||
// },
|
||||
// });
|
||||
|
||||
// if (exists) {
|
||||
// console.log('already exists org');
|
||||
// } else {
|
||||
// const clerkOrgMembers =
|
||||
// await clerkClient.organizations.getOrganizationMembershipList({
|
||||
// organizationId: org.id,
|
||||
// });
|
||||
|
||||
// const members = clerkOrgMembers.data.map((member) => {
|
||||
// const user = users.find(
|
||||
// (u) => u.id === member.publicUserData?.userId
|
||||
// );
|
||||
// return {
|
||||
// userId: member.publicUserData?.userId,
|
||||
// role: member.role,
|
||||
// email: user!.primaryEmailAddress!.emailAddress,
|
||||
// };
|
||||
// });
|
||||
|
||||
// await db.organization.create({
|
||||
// data: {
|
||||
// id: org.slug,
|
||||
// name: org.name,
|
||||
// createdBy: {
|
||||
// connect: {
|
||||
// id: org.createdBy,
|
||||
// },
|
||||
// },
|
||||
// members: {
|
||||
// create: members,
|
||||
// },
|
||||
// },
|
||||
// });
|
||||
|
||||
// const invites =
|
||||
// await clerkClient.organizations.getOrganizationInvitationList({
|
||||
// organizationId: org.id,
|
||||
// status: ['pending'],
|
||||
// });
|
||||
|
||||
// for (const invite of invites.data) {
|
||||
// await db.member.create({
|
||||
// data: {
|
||||
// email: invite.emailAddress,
|
||||
// organizationId: org.slug,
|
||||
// role: invite.role,
|
||||
// userId: null,
|
||||
// meta: {
|
||||
// access: invite.publicMetadata?.access as string[],
|
||||
// invitationId: invite.id,
|
||||
// },
|
||||
// },
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// console.log('org does not have any slug', org);
|
||||
// }
|
||||
// } catch (e) {
|
||||
// console.log('ERROR');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.dir(org, { depth: null });
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// console.log('');
|
||||
// }
|
||||
// }
|
||||
|
||||
// process.exit(0);
|
||||
// }
|
||||
|
||||
// main();
|
||||
|
||||
async function main() {
|
||||
const organization = await db.organization.findUnique({
|
||||
where: {
|
||||
id: 'openpanel-dev',
|
||||
members: {
|
||||
some: {
|
||||
userId: 'user_2cEoI8b1SuEFbZERGEAyVvC676F',
|
||||
},
|
||||
},
|
||||
},
|
||||
include: {
|
||||
members: {
|
||||
select: {
|
||||
role: true,
|
||||
user: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
console.dir(organization, { depth: null });
|
||||
}
|
||||
main();
|
||||
65
apps/api/scripts/migrate-origins.ts
Normal file
65
apps/api/scripts/migrate-origins.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { ch, chQuery } from '@openpanel/db';
|
||||
|
||||
async function main() {
|
||||
const projects = await chQuery(
|
||||
`SELECT distinct project_id FROM events ORDER BY project_id`
|
||||
);
|
||||
const withOrigin = [];
|
||||
|
||||
for (const project of projects) {
|
||||
try {
|
||||
const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([
|
||||
await chQuery(
|
||||
`SELECT * FROM events WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`
|
||||
),
|
||||
await chQuery(
|
||||
`SELECT * FROM events WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`
|
||||
),
|
||||
]);
|
||||
|
||||
if (eventWithOrigin[0] && eventWithoutOrigin[0]) {
|
||||
console.log(`Project ${project.project_id} as events without origin`);
|
||||
console.log(`- Origin: ${eventWithOrigin[0].origin}`);
|
||||
withOrigin.push(project.project_id);
|
||||
const events = await chQuery(
|
||||
`SELECT count(*) as count FROM events WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`
|
||||
);
|
||||
console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);
|
||||
await ch.command({
|
||||
query: `ALTER TABLE events UPDATE origin = '${eventWithOrigin[0].origin}' WHERE project_id = '${project.project_id}' AND path != '' AND origin = '';`,
|
||||
clickhouse_settings: {
|
||||
wait_end_of_query: 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (!eventWithOrigin[0] && eventWithoutOrigin[0]) {
|
||||
console.log(
|
||||
`😧 Project ${project.project_id} has no events with origin (last event ${eventWithoutOrigin[0].created_at})`
|
||||
);
|
||||
console.log('- NO ORIGIN');
|
||||
}
|
||||
|
||||
if (!eventWithOrigin[0] && !eventWithoutOrigin[0]) {
|
||||
console.log(
|
||||
`🔥 WARNING: Project ${project.project_id} has no events at all?!?!?!`
|
||||
);
|
||||
}
|
||||
|
||||
if (eventWithOrigin[0] && !eventWithoutOrigin[0]) {
|
||||
console.log(
|
||||
`✅ Project ${project.project_id} has all events with origin!!!`
|
||||
);
|
||||
}
|
||||
console.log('');
|
||||
console.log('');
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
} catch (e) {
|
||||
console.log('🥵 ERROR ORRROR');
|
||||
console.log('Error for project', project.project_id);
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
main();
|
||||
@@ -5,6 +5,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { generateDeviceId } from '@openpanel/common';
|
||||
import { getSalts } from '@openpanel/db';
|
||||
import { eventsQueue } from '@openpanel/queue';
|
||||
import { redis } from '@openpanel/redis';
|
||||
import type { PostEventPayload } from '@openpanel/sdk';
|
||||
|
||||
export async function postEvent(
|
||||
@@ -63,7 +64,6 @@ export async function postEvent(
|
||||
}
|
||||
const ip = getClientIp(request)!;
|
||||
const ua = request.headers['user-agent']!;
|
||||
const origin = request.headers.origin!;
|
||||
const projectId = request.client?.projectId;
|
||||
|
||||
if (!projectId) {
|
||||
@@ -84,19 +84,15 @@ export async function postEvent(
|
||||
ip,
|
||||
ua,
|
||||
});
|
||||
// TODO: Remove after 2024-09-26
|
||||
const currentDeviceIdDeprecated = generateDeviceId({
|
||||
salt: salts.current,
|
||||
origin,
|
||||
ip,
|
||||
ua,
|
||||
});
|
||||
const previousDeviceIdDeprecated = generateDeviceId({
|
||||
salt: salts.previous,
|
||||
origin,
|
||||
ip,
|
||||
ua,
|
||||
});
|
||||
|
||||
// this will ensure that we don't have multiple events creating sessions
|
||||
const locked = await redis.set(
|
||||
`request:priority:${currentDeviceId}-${previousDeviceId}`,
|
||||
'locked',
|
||||
'EX',
|
||||
10,
|
||||
'NX'
|
||||
);
|
||||
|
||||
eventsQueue.add('event', {
|
||||
type: 'incomingEvent',
|
||||
@@ -105,13 +101,15 @@ export async function postEvent(
|
||||
headers: {
|
||||
ua,
|
||||
},
|
||||
event: request.body,
|
||||
event: {
|
||||
...request.body,
|
||||
// Dont rely on the client for the timestamp
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
geo,
|
||||
currentDeviceId,
|
||||
previousDeviceId,
|
||||
// TODO: Remove after 2024-09-26
|
||||
currentDeviceIdDeprecated,
|
||||
previousDeviceIdDeprecated,
|
||||
priority: locked === 'OK',
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ export function getLiveEventInfo(key: string) {
|
||||
return key.split(':').slice(2) as [string, string];
|
||||
}
|
||||
|
||||
export async function test(
|
||||
export async function testVisitors(
|
||||
req: FastifyRequest<{
|
||||
Params: {
|
||||
projectId: string;
|
||||
@@ -28,13 +28,15 @@ export async function test(
|
||||
reply: FastifyReply
|
||||
) {
|
||||
const events = await getEvents(
|
||||
`SELECT * FROM events WHERE project_id = ${escape(req.params.projectId)} AND name = 'screen_view' LIMIT 500`
|
||||
`SELECT * FROM events LIMIT 500`
|
||||
// `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
|
||||
);
|
||||
const event = events[Math.floor(Math.random() * events.length)];
|
||||
if (!event) {
|
||||
return reply.status(404).send('No event found');
|
||||
}
|
||||
redisPub.publish('event', superjson.stringify(event));
|
||||
event.projectId = req.params.projectId;
|
||||
redisPub.publish('event:received', superjson.stringify(event));
|
||||
redis.set(
|
||||
`live:event:${event.projectId}:${Math.random() * 1000}`,
|
||||
'',
|
||||
@@ -44,6 +46,26 @@ export async function test(
|
||||
reply.status(202).send(event);
|
||||
}
|
||||
|
||||
export async function testEvents(
|
||||
req: FastifyRequest<{
|
||||
Params: {
|
||||
projectId: string;
|
||||
};
|
||||
}>,
|
||||
reply: FastifyReply
|
||||
) {
|
||||
const events = await getEvents(
|
||||
`SELECT * FROM events LIMIT 500`
|
||||
// `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
|
||||
);
|
||||
const event = events[Math.floor(Math.random() * events.length)];
|
||||
if (!event) {
|
||||
return reply.status(404).send('No event found');
|
||||
}
|
||||
redisPub.publish('event:saved', superjson.stringify(event));
|
||||
reply.status(202).send(event);
|
||||
}
|
||||
|
||||
export function wsVisitors(
|
||||
connection: {
|
||||
socket: WebSocket;
|
||||
@@ -56,11 +78,11 @@ export function wsVisitors(
|
||||
) {
|
||||
const { params } = req;
|
||||
|
||||
redisSub.subscribe('event');
|
||||
redisSub.subscribe('event:received');
|
||||
redisSub.psubscribe('__key*:expired');
|
||||
|
||||
const message = (channel: string, message: string) => {
|
||||
if (channel === 'event') {
|
||||
if (channel === 'event:received') {
|
||||
const event = getSuperJson<IServiceCreateEventPayload>(message);
|
||||
if (event?.projectId === params.projectId) {
|
||||
getLiveVisitors(params.projectId).then((count) => {
|
||||
@@ -82,7 +104,7 @@ export function wsVisitors(
|
||||
redisSub.on('pmessage', pmessage);
|
||||
|
||||
connection.socket.on('close', () => {
|
||||
redisSub.unsubscribe('event');
|
||||
redisSub.unsubscribe('event:saved');
|
||||
redisSub.punsubscribe('__key*:expired');
|
||||
redisSub.off('message', message);
|
||||
redisSub.off('pmessage', pmessage);
|
||||
@@ -90,7 +112,7 @@ export function wsVisitors(
|
||||
}
|
||||
|
||||
export function wsEvents(connection: { socket: WebSocket }) {
|
||||
redisSub.subscribe('event');
|
||||
redisSub.subscribe('event:saved');
|
||||
|
||||
const message = (channel: string, message: string) => {
|
||||
const event = getSuperJson<IServiceCreateEventPayload>(message);
|
||||
@@ -102,7 +124,7 @@ export function wsEvents(connection: { socket: WebSocket }) {
|
||||
redisSub.on('message', message);
|
||||
|
||||
connection.socket.on('close', () => {
|
||||
redisSub.unsubscribe('event');
|
||||
redisSub.unsubscribe('event:saved');
|
||||
redisSub.off('message', message);
|
||||
});
|
||||
}
|
||||
@@ -129,7 +151,7 @@ export async function wsProjectEvents(
|
||||
projectId: params.projectId,
|
||||
});
|
||||
|
||||
redisSub.subscribe('event');
|
||||
redisSub.subscribe('event:saved');
|
||||
|
||||
const message = async (channel: string, message: string) => {
|
||||
const event = getSuperJson<IServiceCreateEventPayload>(message);
|
||||
@@ -151,7 +173,7 @@ export async function wsProjectEvents(
|
||||
redisSub.on('message', message as any);
|
||||
|
||||
connection.socket.on('close', () => {
|
||||
redisSub.unsubscribe('event');
|
||||
redisSub.unsubscribe('event:saved');
|
||||
redisSub.off('message', message as any);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { getClientIp, parseIp } from '@/utils/parseIp';
|
||||
import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
|
||||
import { parseUserAgent } from '@/utils/parseUserAgent';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { assocPath, pathOr } from 'ramda';
|
||||
|
||||
@@ -29,7 +29,7 @@ export async function updateProfile(
|
||||
properties: {
|
||||
...(properties ?? {}),
|
||||
...(ip ? geo : {}),
|
||||
...(isUserAgentSet(ua) ? uaInfo : {}),
|
||||
...uaInfo,
|
||||
},
|
||||
...rest,
|
||||
});
|
||||
|
||||
152
apps/api/src/controllers/webhook.controller.ts
Normal file
152
apps/api/src/controllers/webhook.controller.ts
Normal file
@@ -0,0 +1,152 @@
|
||||
import type { WebhookEvent } from '@clerk/fastify';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { pathOr } from 'ramda';
|
||||
import { Webhook } from 'svix';
|
||||
|
||||
import { AccessLevel, db } from '@openpanel/db';
|
||||
|
||||
if (!process.env.CLERK_SIGNING_SECRET) {
|
||||
throw new Error('CLERK_SIGNING_SECRET is required');
|
||||
}
|
||||
|
||||
const wh = new Webhook(process.env.CLERK_SIGNING_SECRET);
|
||||
|
||||
function verify(body: any, headers: FastifyRequest['headers']) {
|
||||
try {
|
||||
const svix_id = headers['svix-id'] as string;
|
||||
const svix_timestamp = headers['svix-timestamp'] as string;
|
||||
const svix_signature = headers['svix-signature'] as string;
|
||||
|
||||
wh.verify(JSON.stringify(body), {
|
||||
'svix-id': svix_id,
|
||||
'svix-timestamp': svix_timestamp,
|
||||
'svix-signature': svix_signature,
|
||||
});
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function clerkWebhook(
|
||||
request: FastifyRequest<{
|
||||
Body: WebhookEvent;
|
||||
}>,
|
||||
reply: FastifyReply
|
||||
) {
|
||||
const payload = request.body;
|
||||
const verified = verify(payload, request.headers);
|
||||
|
||||
if (!verified) {
|
||||
return reply.send({ message: 'Invalid signature' });
|
||||
}
|
||||
|
||||
if (payload.type === 'user.created') {
|
||||
const email = payload.data.email_addresses[0]?.email_address;
|
||||
|
||||
if (!email) {
|
||||
return Response.json(
|
||||
{ message: 'No email address found' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const user = await db.user.create({
|
||||
data: {
|
||||
id: payload.data.id,
|
||||
email,
|
||||
firstName: payload.data.first_name,
|
||||
lastName: payload.data.last_name,
|
||||
},
|
||||
});
|
||||
|
||||
const memberships = await db.member.findMany({
|
||||
where: {
|
||||
email,
|
||||
userId: null,
|
||||
},
|
||||
});
|
||||
|
||||
for (const membership of memberships) {
|
||||
const access = pathOr<string[]>([], ['meta', 'access'], membership);
|
||||
db.$transaction([
|
||||
// Update the member to link it to the user
|
||||
// This will remove the item from invitations
|
||||
db.member.update({
|
||||
where: {
|
||||
id: membership.id,
|
||||
},
|
||||
data: {
|
||||
userId: user.id,
|
||||
},
|
||||
}),
|
||||
db.projectAccess.createMany({
|
||||
data: access
|
||||
.filter((a) => typeof a === 'string')
|
||||
.map((projectId) => ({
|
||||
organizationSlug: membership.organizationId,
|
||||
organizationId: membership.organizationId,
|
||||
projectId: projectId,
|
||||
userId: user.id,
|
||||
level: AccessLevel.read,
|
||||
})),
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
if (payload.type === 'organizationMembership.created') {
|
||||
const access = payload.data.public_metadata.access;
|
||||
if (Array.isArray(access)) {
|
||||
await db.projectAccess.createMany({
|
||||
data: access
|
||||
.filter((a): a is string => typeof a === 'string')
|
||||
.map((projectId) => ({
|
||||
organizationSlug: payload.data.organization.slug,
|
||||
organizationId: payload.data.organization.slug,
|
||||
projectId: projectId,
|
||||
userId: payload.data.public_user_data.user_id,
|
||||
level: AccessLevel.read,
|
||||
})),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (payload.type === 'user.deleted') {
|
||||
await db.$transaction([
|
||||
db.user.update({
|
||||
where: {
|
||||
id: payload.data.id,
|
||||
},
|
||||
data: {
|
||||
deletedAt: new Date(),
|
||||
firstName: null,
|
||||
lastName: null,
|
||||
email: `deleted+${payload.data.id}@openpanel.dev`,
|
||||
},
|
||||
}),
|
||||
db.projectAccess.deleteMany({
|
||||
where: {
|
||||
userId: payload.data.id,
|
||||
},
|
||||
}),
|
||||
db.member.deleteMany({
|
||||
where: {
|
||||
userId: payload.data.id,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
if (payload.type === 'organizationMembership.deleted') {
|
||||
await db.projectAccess.deleteMany({
|
||||
where: {
|
||||
organizationSlug: payload.data.organization.slug,
|
||||
userId: payload.data.public_user_data.user_id,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
reply.send({ success: true });
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import exportRouter from './routes/export.router';
|
||||
import liveRouter from './routes/live.router';
|
||||
import miscRouter from './routes/misc.router';
|
||||
import profileRouter from './routes/profile.router';
|
||||
import webhookRouter from './routes/webhook.router';
|
||||
import { logger, logInfo } from './utils/logger';
|
||||
|
||||
declare module 'fastify' {
|
||||
@@ -82,6 +83,7 @@ const startServer = async () => {
|
||||
fastify.register(profileRouter, { prefix: '/profile' });
|
||||
fastify.register(miscRouter, { prefix: '/misc' });
|
||||
fastify.register(exportRouter, { prefix: '/export' });
|
||||
fastify.register(webhookRouter, { prefix: '/webhook' });
|
||||
fastify.setErrorHandler((error) => {
|
||||
logger.error(error, 'Error in request');
|
||||
});
|
||||
@@ -89,35 +91,45 @@ const startServer = async () => {
|
||||
reply.send({ name: 'openpanel sdk api' });
|
||||
});
|
||||
fastify.get('/healthcheck', async (request, reply) => {
|
||||
try {
|
||||
const redisRes = await withTimings(redis.keys('*'));
|
||||
const dbRes = await withTimings(db.project.findFirst());
|
||||
const queueRes = await withTimings(eventsQueue.getCompleted());
|
||||
const chRes = await withTimings(
|
||||
chQuery('SELECT * FROM events LIMIT 1')
|
||||
);
|
||||
const redisRes = await withTimings(redis.keys('*')).catch(
|
||||
(e: Error) => e
|
||||
);
|
||||
const dbRes = await withTimings(db.project.findFirst()).catch(
|
||||
(e: Error) => e
|
||||
);
|
||||
const queueRes = await withTimings(eventsQueue.getCompleted()).catch(
|
||||
(e: Error) => e
|
||||
);
|
||||
const chRes = await withTimings(
|
||||
chQuery('SELECT * FROM events LIMIT 1')
|
||||
).catch((e: Error) => e);
|
||||
|
||||
reply.send({
|
||||
redis: {
|
||||
ok: redisRes[1].length ? true : false,
|
||||
time: `${redisRes[0]}ms`,
|
||||
},
|
||||
db: {
|
||||
ok: dbRes[1] ? true : false,
|
||||
time: `${dbRes[0]}ms`,
|
||||
},
|
||||
queue: {
|
||||
ok: queueRes[1].length ? true : false,
|
||||
time: `${queueRes[0]}ms`,
|
||||
},
|
||||
ch: {
|
||||
ok: chRes[1].length ? true : false,
|
||||
time: `${chRes[0]}ms`,
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
reply.status(500).send();
|
||||
}
|
||||
reply.send({
|
||||
redis: Array.isArray(redisRes)
|
||||
? {
|
||||
ok: redisRes[1].length ? true : false,
|
||||
time: `${redisRes[0]}ms`,
|
||||
}
|
||||
: redisRes,
|
||||
db: Array.isArray(dbRes)
|
||||
? {
|
||||
ok: dbRes[1] ? true : false,
|
||||
time: `${dbRes[0]}ms`,
|
||||
}
|
||||
: dbRes,
|
||||
queue: Array.isArray(queueRes)
|
||||
? {
|
||||
ok: queueRes[1].length ? true : false,
|
||||
time: `${queueRes[0]}ms`,
|
||||
}
|
||||
: queueRes,
|
||||
ch: Array.isArray(chRes)
|
||||
? {
|
||||
ok: chRes[1].length ? true : false,
|
||||
time: `${chRes[0]}ms`,
|
||||
}
|
||||
: chRes,
|
||||
});
|
||||
});
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
for (const signal of ['SIGINT', 'SIGTERM']) {
|
||||
|
||||
@@ -3,10 +3,15 @@ import fastifyWS from '@fastify/websocket';
|
||||
import type { FastifyPluginCallback } from 'fastify';
|
||||
|
||||
const liveRouter: FastifyPluginCallback = (fastify, opts, done) => {
|
||||
fastify.route({
|
||||
method: 'GET',
|
||||
url: '/visitors/test/:projectId',
|
||||
handler: controller.testVisitors,
|
||||
});
|
||||
fastify.route({
|
||||
method: 'GET',
|
||||
url: '/events/test/:projectId',
|
||||
handler: controller.test,
|
||||
handler: controller.testEvents,
|
||||
});
|
||||
|
||||
fastify.register(fastifyWS);
|
||||
|
||||
13
apps/api/src/routes/webhook.router.ts
Normal file
13
apps/api/src/routes/webhook.router.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import * as controller from '@/controllers/webhook.controller';
|
||||
import type { FastifyPluginCallback } from 'fastify';
|
||||
|
||||
const webhookRouter: FastifyPluginCallback = (fastify, opts, done) => {
|
||||
fastify.route({
|
||||
method: 'POST',
|
||||
url: '/clerk',
|
||||
handler: controller.clerkWebhook,
|
||||
});
|
||||
done();
|
||||
};
|
||||
|
||||
export default webhookRouter;
|
||||
@@ -5,8 +5,6 @@ import { verifyPassword } from '@openpanel/common';
|
||||
import type { Client, IServiceClient } from '@openpanel/db';
|
||||
import { ClientType, db } from '@openpanel/db';
|
||||
|
||||
import { logger } from './logger';
|
||||
|
||||
const cleanDomain = (domain: string) =>
|
||||
domain
|
||||
.replace('www.', '')
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
import { UAParser } from 'ua-parser-js';
|
||||
|
||||
export function isUserAgentSet(ua: string) {
|
||||
return ua !== 'node' && ua !== 'undici' && !!ua;
|
||||
}
|
||||
const parsedServerUa = {
|
||||
isServer: true,
|
||||
device: 'server',
|
||||
} as const;
|
||||
|
||||
export function parseUserAgent(ua: string) {
|
||||
export function parseUserAgent(ua?: string | null) {
|
||||
if (!ua) return parsedServerUa;
|
||||
const res = new UAParser(ua).getResult();
|
||||
|
||||
if (isServer(ua)) return parsedServerUa;
|
||||
|
||||
return {
|
||||
os: res.os.name,
|
||||
osVersion: res.os.version,
|
||||
@@ -14,20 +19,86 @@ export function parseUserAgent(ua: string) {
|
||||
device: res.device.type ?? getDevice(ua),
|
||||
brand: res.device.vendor,
|
||||
model: res.device.model,
|
||||
};
|
||||
isServer: false,
|
||||
} as const;
|
||||
}
|
||||
|
||||
const userAgentServerList: string[] = [
|
||||
// Node.js libraries
|
||||
'node',
|
||||
'node-fetch',
|
||||
'axios',
|
||||
'request',
|
||||
'superagent',
|
||||
'undici',
|
||||
|
||||
// Python libraries
|
||||
'python-requests',
|
||||
'python-urllib',
|
||||
|
||||
// Ruby libraries
|
||||
'Faraday',
|
||||
'Ruby',
|
||||
'http.rb',
|
||||
|
||||
// Go libraries
|
||||
'Go-http-client',
|
||||
'Go-http-client',
|
||||
|
||||
// Java libraries
|
||||
'Apache-HttpClient',
|
||||
'okhttp',
|
||||
'okhowtp',
|
||||
|
||||
// PHP libraries
|
||||
'GuzzleHttp',
|
||||
'PHP-cURL',
|
||||
|
||||
// Other
|
||||
'Dart',
|
||||
'RestSharp', // Popular .NET HTTP client library
|
||||
'HttpClientFactory', // .NET's typed client factory
|
||||
'Ktor', // A client for Kotlin
|
||||
'Ning', // Async HTTP client for Java
|
||||
'grpc-csharp', // gRPC for C#
|
||||
'Volley', // HTTP library used in Android apps for making network requests
|
||||
'Spring',
|
||||
'vert.x',
|
||||
'grpc-',
|
||||
];
|
||||
|
||||
function isServer(userAgent: string) {
|
||||
const match = userAgentServerList.some((server) =>
|
||||
userAgent.includes(server)
|
||||
);
|
||||
if (match) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return !!userAgent.match(/^[^\s]+\/[\d.]+$/);
|
||||
}
|
||||
|
||||
export function getDevice(ua: string) {
|
||||
const t1 =
|
||||
const mobile1 =
|
||||
/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
|
||||
ua
|
||||
);
|
||||
const t2 =
|
||||
const mobile2 =
|
||||
/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
|
||||
ua.slice(0, 4)
|
||||
);
|
||||
if (t1 || t2) {
|
||||
const tablet =
|
||||
/tablet|ipad|android(?!.*mobile)|xoom|sch-i800|kindle|silk|playbook/i.test(
|
||||
ua
|
||||
);
|
||||
|
||||
if (mobile1 || mobile2) {
|
||||
return 'mobile';
|
||||
}
|
||||
|
||||
if (tablet) {
|
||||
return 'tablet';
|
||||
}
|
||||
|
||||
return 'desktop';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user