improve(api): update api to fastify v5

This commit is contained in:
Carl-Gerhard Lindesvärd
2025-02-28 09:19:27 +01:00
parent f0b0f27a8f
commit 7750ca117f
25 changed files with 520 additions and 1642 deletions

View File

@@ -11,11 +11,11 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@fastify/compress": "^7.0.3",
"@fastify/cookie": "^9.3.1",
"@fastify/cors": "^9.0.0",
"@fastify/rate-limit": "^9.1.0",
"@fastify/websocket": "^8.3.1",
"@fastify/compress": "^8.0.1",
"@fastify/cookie": "^11.0.2",
"@fastify/cors": "^11.0.0",
"@fastify/rate-limit": "^10.2.2",
"@fastify/websocket": "^11.0.2",
"@node-rs/argon2": "^2.0.2",
"@openpanel/auth": "workspace:^",
"@openpanel/common": "workspace:*",
@@ -28,21 +28,22 @@
"@openpanel/redis": "workspace:*",
"@openpanel/trpc": "workspace:*",
"@openpanel/validation": "workspace:*",
"@trpc/server": "^10.45.1",
"@trpc/server": "^10.45.2",
"bcrypt": "^5.1.1",
"fastify": "^4.25.2",
"fastify-metrics": "^11.0.0",
"fastify-raw-body": "^4.2.1",
"ico-to-png": "^0.2.1",
"fast-json-stable-hash": "^1.0.3",
"fastify": "^5.2.1",
"fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0",
"ico-to-png": "^0.2.2",
"jsonwebtoken": "^9.0.2",
"ramda": "^0.29.1",
"request-ip": "^3.3.0",
"sharp": "^0.33.2",
"sharp": "^0.33.5",
"source-map-support": "^0.5.21",
"sqlstring": "^2.3.3",
"superjson": "^1.13.3",
"svix": "^1.24.0",
"url-metadata": "^4.1.0",
"url-metadata": "^4.1.1",
"uuid": "^9.0.1",
"zod": "^3.22.4"
},
@@ -51,12 +52,12 @@
"@openpanel/sdk": "workspace:*",
"@openpanel/tsconfig": "workspace:*",
"@types/js-yaml": "^4.0.9",
"@types/jsonwebtoken": "^9.0.6",
"@types/ramda": "^0.29.6",
"@types/jsonwebtoken": "^9.0.9",
"@types/ramda": "^0.30.2",
"@types/request-ip": "^0.0.41",
"@types/source-map-support": "^0.5.10",
"@types/sqlstring": "^2.3.2",
"@types/uuid": "^9.0.8",
"@types/uuid": "^10.0.0",
"@types/ws": "^8.5.14",
"js-yaml": "^4.1.0",
"tsup": "^7.2.0",

View File

@@ -4,7 +4,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import { generateDeviceId } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { eventsQueue } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import { getLock } from '@openpanel/redis';
import type { PostEventPayload } from '@openpanel/sdk';
import { getStringHeaders, getTimestamp } from './track.controller';
@@ -42,12 +42,10 @@ export async function postEvent(
const isScreenView = request.body.name === 'screen_view';
// this will ensure that we don't have multiple events creating sessions
const LOCK_DURATION = 1000;
const locked = await getRedisCache().set(
const locked = await getLock(
`request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,
'locked',
'PX',
LOCK_DURATION,
'NX',
);
await eventsQueue.add(
@@ -65,7 +63,7 @@ export async function postEvent(
geo,
currentDeviceId,
previousDeviceId,
priority: locked === 'OK',
priority: locked,
},
},
{

View File

@@ -1,8 +1,7 @@
import type { SocketStream } from '@fastify/websocket';
import type { FastifyRequest } from 'fastify';
import superjson from 'superjson';
import type { WebSocket } from 'ws';
import type { WebSocket } from '@fastify/websocket';
import {
eventBuffer,
getProfileByIdCached,
@@ -16,20 +15,12 @@ import {
import { getProjectAccess } from '@openpanel/trpc';
import { getOrganizationAccess } from '@openpanel/trpc/src/access';
type WebSocketConnection = SocketStream & {
socket: WebSocket & {
on(event: 'close', listener: () => void): void;
send(data: string): void;
close(): void;
};
};
export function getLiveEventInfo(key: string) {
return key.split(':').slice(2) as [string, string];
}
export function wsVisitors(
connection: WebSocketConnection,
socket: WebSocket,
req: FastifyRequest<{
Params: {
projectId: string;
@@ -37,11 +28,10 @@ export function wsVisitors(
}>,
) {
const { params } = req;
const unsubscribe = subscribeToPublishedEvent('events', 'saved', (event) => {
if (event?.projectId === params.projectId) {
eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
connection.socket.send(String(count));
socket.send(String(count));
});
}
});
@@ -52,20 +42,20 @@ export function wsVisitors(
const [projectId] = getLiveEventInfo(key);
if (projectId && projectId === params.projectId) {
eventBuffer.getActiveVisitorCount(params.projectId).then((count) => {
connection.socket.send(String(count));
socket.send(String(count));
});
}
},
);
connection.socket.on('close', () => {
socket.on('close', () => {
unsubscribe();
punsubscribe();
});
}
export async function wsProjectEvents(
connection: WebSocketConnection,
socket: WebSocket,
req: FastifyRequest<{
Params: {
projectId: string;
@@ -80,15 +70,15 @@ export async function wsProjectEvents(
const type = query.type || 'saved';
if (!['saved', 'received'].includes(type)) {
connection.socket.send('Invalid type');
connection.socket.close();
socket.send('Invalid type');
socket.close();
return;
}
const userId = req.session?.userId;
if (!userId) {
connection.socket.send('No active session');
connection.socket.close();
socket.send('No active session');
socket.close();
return;
}
@@ -106,7 +96,7 @@ export async function wsProjectEvents(
event.profileId,
event.projectId,
);
connection.socket.send(
socket.send(
superjson.stringify(
access
? {
@@ -120,11 +110,11 @@ export async function wsProjectEvents(
},
);
connection.socket.on('close', () => unsubscribe());
socket.on('close', () => unsubscribe());
}
export async function wsProjectNotifications(
connection: WebSocketConnection,
socket: WebSocket,
req: FastifyRequest<{
Params: {
projectId: string;
@@ -135,8 +125,8 @@ export async function wsProjectNotifications(
const userId = req.session?.userId;
if (!userId) {
connection.socket.send('No active session');
connection.socket.close();
socket.send('No active session');
socket.close();
return;
}
@@ -146,8 +136,8 @@ export async function wsProjectNotifications(
});
if (!access) {
connection.socket.send('No access');
connection.socket.close();
socket.send('No access');
socket.close();
return;
}
@@ -156,16 +146,16 @@ export async function wsProjectNotifications(
'created',
(notification) => {
if (notification.projectId === params.projectId) {
connection.socket.send(superjson.stringify(notification));
socket.send(superjson.stringify(notification));
}
},
);
connection.socket.on('close', () => unsubscribe());
socket.on('close', () => unsubscribe());
}
export async function wsOrganizationEvents(
connection: WebSocketConnection,
socket: WebSocket,
req: FastifyRequest<{
Params: {
organizationId: string;
@@ -176,8 +166,8 @@ export async function wsOrganizationEvents(
const userId = req.session?.userId;
if (!userId) {
connection.socket.send('No active session');
connection.socket.close();
socket.send('No active session');
socket.close();
return;
}
@@ -187,8 +177,8 @@ export async function wsOrganizationEvents(
});
if (!access) {
connection.socket.send('No access');
connection.socket.close();
socket.send('No access');
socket.close();
return;
}
@@ -196,9 +186,9 @@ export async function wsOrganizationEvents(
'organization',
'subscription_updated',
(message) => {
connection.socket.send(setSuperJson(message));
socket.send(setSuperJson(message));
},
);
connection.socket.on('close', () => unsubscribe());
socket.on('close', () => unsubscribe());
}

View File

@@ -76,7 +76,7 @@ async function handleExistingUser({
sessionToken,
session.expiresAt,
);
return reply.status(302).redirect(process.env.NEXT_PUBLIC_DASHBOARD_URL!);
return reply.redirect(process.env.NEXT_PUBLIC_DASHBOARD_URL!);
}
async function handleNewUser({
@@ -138,7 +138,7 @@ async function handleNewUser({
sessionToken,
session.expiresAt,
);
return reply.status(302).redirect(process.env.NEXT_PUBLIC_DASHBOARD_URL!);
return reply.redirect(process.env.NEXT_PUBLIC_DASHBOARD_URL!);
}
// Provider-specific user fetching

View File

@@ -6,7 +6,7 @@ import { path, assocPath, pathOr, pick } from 'ramda';
import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
import { getProfileById, getSalts, upsertProfile } from '@openpanel/db';
import { eventsQueue } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import { getLock } from '@openpanel/redis';
import type {
DecrementPayload,
IdentifyPayload,
@@ -230,12 +230,10 @@ async function track({
const isScreenView = payload.name === 'screen_view';
// this will ensure that we don't have multiple events creating sessions
const LOCK_DURATION = 1000;
const locked = await getRedisCache().set(
const locked = await getLock(
`request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,
'locked',
'PX',
LOCK_DURATION,
'NX',
);
await eventsQueue.add(
@@ -253,7 +251,7 @@ async function track({
geo,
currentDeviceId,
previousDeviceId,
priority: locked === 'OK',
priority: locked,
},
},
{

View File

@@ -0,0 +1,21 @@
import { getLock } from '@openpanel/redis';
import fastJsonStableHash from 'fast-json-stable-hash';
import type { FastifyReply, FastifyRequest } from 'fastify';
export async function deduplicateHook(
request: FastifyRequest,
reply: FastifyReply,
) {
if (typeof request.body === 'object') {
const locked = await getLock(
`fastify:deduplicate:${fastJsonStableHash.hash(request.body, 'md5')}`,
'1',
100,
);
if (locked) {
return;
}
}
reply.status(200).send('Duplicated event');
}

View File

@@ -1,14 +1,6 @@
import type {
FastifyReply,
FastifyRequest,
HookHandlerDoneFunction,
} from 'fastify';
import type { FastifyRequest } from 'fastify';
export function fixHook(
request: FastifyRequest,
reply: FastifyReply,
done: HookHandlerDoneFunction,
) {
export async function fixHook(request: FastifyRequest) {
const ua = request.headers['user-agent'];
// Swift SDK issue: https://github.com/Openpanel-dev/swift-sdk/commit/d588fa761a36a33f3b78eb79d83bfd524e3c7144
if (ua) {
@@ -21,5 +13,4 @@ export function fixHook(
);
}
}
done();
}

View File

@@ -5,14 +5,9 @@ import type {
HookHandlerDoneFunction,
} from 'fastify';
export function ipHook(
request: FastifyRequest,
reply: FastifyReply,
done: HookHandlerDoneFunction,
) {
export async function ipHook(request: FastifyRequest) {
const ip = getClientIp(request);
if (ip) {
request.clientIp = ip;
}
done();
}

View File

@@ -4,13 +4,8 @@ import type {
HookHandlerDoneFunction,
} from 'fastify';
export function requestIdHook(
request: FastifyRequest,
reply: FastifyReply,
done: HookHandlerDoneFunction,
) {
export async function requestIdHook(request: FastifyRequest) {
if (!request.headers['request-id']) {
request.headers['request-id'] = request.id;
}
done();
}

View File

@@ -1,8 +1,4 @@
import type {
FastifyReply,
FastifyRequest,
HookHandlerDoneFunction,
} from 'fastify';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { path, pick } from 'ramda';
const ignoreLog = ['/healthcheck', '/metrics', '/misc'];
@@ -19,16 +15,15 @@ const getTrpcInput = (
}
};
export function requestLoggingHook(
export async function requestLoggingHook(
request: FastifyRequest,
reply: FastifyReply,
done: HookHandlerDoneFunction,
) {
if (ignoreMethods.includes(request.method)) {
return done();
return;
}
if (ignoreLog.some((path) => request.url.startsWith(path))) {
return done();
return;
}
if (request.url.includes('trpc')) {
request.log.info('request done', {
@@ -54,5 +49,4 @@ export function requestLoggingHook(
body: request.body,
});
}
done();
}

View File

@@ -1,14 +1,5 @@
import type {
FastifyReply,
FastifyRequest,
HookHandlerDoneFunction,
} from 'fastify';
import type { FastifyRequest } from 'fastify';
export function timestampHook(
request: FastifyRequest,
reply: FastifyReply,
done: HookHandlerDoneFunction,
) {
export async function timestampHook(request: FastifyRequest) {
request.timestamp = Date.now();
done();
}

View File

@@ -1,4 +1,3 @@
import zlib from 'node:zlib';
import compress from '@fastify/compress';
import cookie from '@fastify/cookie';
import cors, { type FastifyCorsOptions } from '@fastify/cors';
@@ -59,7 +58,7 @@ const startServer = async () => {
const fastify = Fastify({
maxParamLength: 15_000,
bodyLimit: 1048576 * 500, // 500MB
logger: logger as unknown as FastifyBaseLogger,
loggerInstance: logger as unknown as FastifyBaseLogger,
disableRequestLogging: true,
genReqId: (req) =>
req.headers['request-id']
@@ -107,61 +106,26 @@ const startServer = async () => {
encodings: ['gzip', 'deflate'],
});
fastify.addContentTypeParser(
'application/json',
{ parseAs: 'buffer' },
(req, body, done) => {
const isGzipped = req.headers['content-encoding'] === 'gzip';
if (isGzipped) {
zlib.gunzip(body, (err, decompressedBody) => {
if (err) {
done(err);
} else {
try {
const json = JSON.parse(decompressedBody.toString());
done(null, json);
} catch (parseError) {
done(new Error('Invalid JSON'));
}
}
});
} else {
try {
const json = JSON.parse(body.toString());
done(null, json);
} catch (parseError) {
done(new Error('Invalid JSON'));
}
}
},
);
// Dashboard API
fastify.register((instance, opts, done) => {
fastify.register(async (instance) => {
instance.register(cookie, {
secret: process.env.COOKIE_SECRET ?? '',
hook: 'onRequest',
parseOptions: {},
});
instance.addHook('onRequest', (req, reply, done) => {
instance.addHook('onRequest', async (req) => {
if (req.cookies?.session) {
validateSessionToken(req.cookies.session)
.then((session) => {
if (session.session) {
req.session = session;
}
})
.catch(() => {
req.session = EMPTY_SESSION;
})
.finally(() => {
done();
});
try {
const session = await validateSessionToken(req.cookies.session);
if (session.session) {
req.session = session;
}
} catch (e) {
req.session = EMPTY_SESSION;
}
} else {
req.session = EMPTY_SESSION;
done();
}
});
@@ -184,11 +148,10 @@ const startServer = async () => {
instance.register(webhookRouter, { prefix: '/webhook' });
instance.register(oauthRouter, { prefix: '/oauth' });
instance.register(miscRouter, { prefix: '/misc' });
done();
});
// Public API
fastify.register((instance, opts, done) => {
fastify.register(async (instance) => {
instance.register(metricsPlugin, { endpoint: '/metrics' });
instance.register(eventRouter, { prefix: '/event' });
instance.register(profileRouter, { prefix: '/profile' });
@@ -200,7 +163,6 @@ const startServer = async () => {
instance.get('/', (_request, reply) =>
reply.send({ name: 'openpanel sdk api' }),
);
done();
});
fastify.setErrorHandler((error, request, reply) => {

View File

@@ -2,9 +2,11 @@ import * as controller from '@/controllers/event.controller';
import type { FastifyPluginCallback } from 'fastify';
import { clientHook } from '@/hooks/client.hook';
import { deduplicateHook } from '@/hooks/deduplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
const eventRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preHandler', deduplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.addHook('preHandler', isBotHook);
@@ -13,7 +15,6 @@ const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
url: '/',
handler: controller.postEvent,
});
done();
};
export default eventRouter;

View File

@@ -4,7 +4,7 @@ import { activateRateLimiter } from '@/utils/rate-limiter';
import { Prisma } from '@openpanel/db';
import type { FastifyPluginCallback, FastifyRequest } from 'fastify';
const exportRouter: FastifyPluginCallback = async (fastify, opts, done) => {
const exportRouter: FastifyPluginCallback = async (fastify) => {
await activateRateLimiter({
fastify,
max: 10,
@@ -46,7 +46,6 @@ const exportRouter: FastifyPluginCallback = async (fastify, opts, done) => {
url: '/charts',
handler: controller.charts,
});
done();
};
export default exportRouter;

View File

@@ -4,7 +4,7 @@ import type { FastifyPluginCallback, FastifyRequest } from 'fastify';
import { Prisma } from '@openpanel/db';
const importRouter: FastifyPluginCallback = (fastify, opts, done) => {
const importRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preHandler', async (req: FastifyRequest, reply) => {
try {
const client = await validateImportRequest(req.headers);
@@ -34,8 +34,6 @@ const importRouter: FastifyPluginCallback = (fastify, opts, done) => {
url: '/events',
handler: controller.importEvents,
});
done();
};
export default importRouter;

View File

@@ -2,36 +2,31 @@ import * as controller from '@/controllers/live.controller';
import fastifyWS from '@fastify/websocket';
import type { FastifyPluginCallback } from 'fastify';
// TODO: `as any` is a workaround since it starts to break after changed module resolution to bundler
// which is needed for @polar/sdk (dont have time to resolve this now)
const liveRouter: FastifyPluginCallback = (fastify, opts, done) => {
const liveRouter: FastifyPluginCallback = async (fastify) => {
fastify.register(fastifyWS);
fastify.register((fastify, _, done) => {
fastify.register(async (fastify) => {
fastify.get(
'/organization/:organizationId',
{ websocket: true },
controller.wsOrganizationEvents as any,
controller.wsOrganizationEvents,
);
fastify.get(
'/visitors/:projectId',
{ websocket: true },
controller.wsVisitors as any,
controller.wsVisitors,
);
fastify.get(
'/events/:projectId',
{ websocket: true },
controller.wsProjectEvents as any,
controller.wsProjectEvents,
);
fastify.get(
'/notifications/:projectId',
{ websocket: true },
controller.wsProjectNotifications as any,
controller.wsProjectNotifications,
);
done();
});
done();
};
export default liveRouter;

View File

@@ -1,7 +1,7 @@
import * as controller from '@/controllers/misc.controller';
import type { FastifyPluginCallback } from 'fastify';
const miscRouter: FastifyPluginCallback = (fastify, opts, done) => {
const miscRouter: FastifyPluginCallback = async (fastify) => {
fastify.route({
method: 'POST',
url: '/ping',
@@ -25,8 +25,6 @@ const miscRouter: FastifyPluginCallback = (fastify, opts, done) => {
url: '/favicon/clear',
handler: controller.clearFavicons,
});
done();
};
export default miscRouter;

View File

@@ -1,7 +1,7 @@
import * as controller from '@/controllers/oauth-callback.controller';
import type { FastifyPluginCallback } from 'fastify';
const router: FastifyPluginCallback = (fastify, opts, done) => {
const router: FastifyPluginCallback = async (fastify) => {
fastify.route({
method: 'GET',
url: '/github/callback',
@@ -12,7 +12,6 @@ const router: FastifyPluginCallback = (fastify, opts, done) => {
url: '/google/callback',
handler: controller.googleCallback,
});
done();
};
export default router;

View File

@@ -1,9 +1,11 @@
import * as controller from '@/controllers/profile.controller';
import { clientHook } from '@/hooks/client.hook';
import { deduplicateHook } from '@/hooks/deduplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
import type { FastifyPluginCallback } from 'fastify';
const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
const eventRouter: FastifyPluginCallback = async (fastify) => {
fastify.addHook('preHandler', deduplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.addHook('preHandler', isBotHook);
@@ -24,7 +26,6 @@ const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
url: '/decrement',
handler: controller.decrementProfileProperty,
});
done();
};
export default eventRouter;

View File

@@ -2,9 +2,11 @@ import { handler } from '@/controllers/track.controller';
import type { FastifyPluginCallback } from 'fastify';
import { clientHook } from '@/hooks/client.hook';
import { deduplicateHook } from '@/hooks/deduplicate.hook';
import { isBotHook } from '@/hooks/is-bot.hook';
const trackRouter: FastifyPluginCallback = (fastify, opts, done) => {
const trackRouter: FastifyPluginCallback = (fastify) => {
fastify.addHook('preHandler', deduplicateHook);
fastify.addHook('preHandler', clientHook);
fastify.addHook('preHandler', isBotHook);
@@ -29,8 +31,6 @@ const trackRouter: FastifyPluginCallback = (fastify, opts, done) => {
},
},
});
done();
};
export default trackRouter;

View File

@@ -1,7 +1,7 @@
import * as controller from '@/controllers/webhook.controller';
import type { FastifyPluginCallback } from 'fastify';
const webhookRouter: FastifyPluginCallback = (fastify, opts, done) => {
const webhookRouter: FastifyPluginCallback = async (fastify) => {
fastify.route({
method: 'GET',
url: '/slack',
@@ -15,7 +15,6 @@ const webhookRouter: FastifyPluginCallback = (fastify, opts, done) => {
rawBody: true,
},
});
done();
};
export default webhookRouter;

View File

@@ -51,10 +51,10 @@
"@tailwindcss/typography": "^0.5.15",
"@tanstack/react-query": "^4.36.1",
"@tanstack/react-table": "^8.11.8",
"@trpc/client": "^10.45.1",
"@trpc/next": "^10.45.1",
"@trpc/react-query": "^10.45.1",
"@trpc/server": "^10.45.1",
"@trpc/client": "^10.45.2",
"@trpc/next": "^10.45.2",
"@trpc/react-query": "^10.45.2",
"@trpc/server": "^10.45.2",
"@types/d3": "^7.4.3",
"bcrypt": "^5.1.1",
"bind-event-listener": "^3.0.0",

View File

@@ -64,3 +64,8 @@ export function getRedisQueue() {
return redisQueue;
}
export async function getLock(key: string, value: string, timeout: number) {
const lock = await getRedisCache().set(key, value, 'PX', timeout, 'NX');
return lock === 'OK';
}

View File

@@ -17,8 +17,8 @@
"@openpanel/validation": "workspace:*",
"@seventy-seven/sdk": "0.0.0-beta.2",
"@trpc-limiter/redis": "^0.0.2",
"@trpc/server": "^10.45.1",
"@trpc/client": "^10.45.1",
"@trpc/server": "^10.45.2",
"@trpc/client": "^10.45.2",
"bcrypt": "^5.1.1",
"date-fns": "^3.3.1",
"mathjs": "^12.3.2",

1845
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff