wip

apps/api/src/controllers/track.controller.ts

@@ -5,11 +5,9 @@ import { HttpError } from '@/utils/errors';
 import { generateId, slug } from '@openpanel/common';
 import { generateDeviceId, parseUserAgent } from '@openpanel/common/server';
 import {
-  TABLE_NAMES,
-  ch,
   getProfileById,
   getSalts,
-  sessionBuffer,
+  replayBuffer,
   upsertProfile,
 } from '@openpanel/db';
 import { type GeoLocation, getGeoLocation } from '@openpanel/geo';
@@ -328,12 +326,7 @@ async function handleReplay(
     is_full_snapshot: payload.is_full_snapshot,
     payload: payload.payload,
   };
-  await ch.insert({
-    table: TABLE_NAMES.session_replay_chunks,
-    values: [row],
-    format: 'JSONEachRow',
-  });
-  await sessionBuffer.markHasReplay(row.session_id);
+  await replayBuffer.add(row);
 }
 
 export async function handler(

apps/api/src/hooks/duplicate.hook.ts

@@ -1,20 +1,20 @@
-import { isDuplicatedEvent } from '@/utils/deduplicate';
 import type {
   DeprecatedPostEventPayload,
   ITrackHandlerPayload,
 } from '@openpanel/validation';
 import type { FastifyReply, FastifyRequest } from 'fastify';
+import { isDuplicatedEvent } from '@/utils/deduplicate';
 
 export async function duplicateHook(
   req: FastifyRequest<{
     Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
   }>,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   const ip = req.clientIp;
   const origin = req.headers.origin;
   const clientId = req.headers['openpanel-client-id'];
-  const shouldCheck = ip && origin && clientId;
+  const shouldCheck = ip && origin && clientId && req.body.type !== 'replay';
 
   const isDuplicate = shouldCheck
     ? await isDuplicatedEvent({
@@ -25,6 +25,7 @@ export async function duplicateHook(
       })
     : false;
 
+  console.log('Duplicate event', isDuplicate);
   if (isDuplicate) {
     return reply.status(200).send('Duplicate event');
   }

apps/api/src/hooks/request-logging.hook.ts

@@ -1,4 +1,3 @@
-import { DEFAULT_IP_HEADER_ORDER } from '@openpanel/common';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { path, pick } from 'ramda';
 
@@ -6,7 +5,7 @@ const ignoreLog = ['/healthcheck', '/healthz', '/metrics', '/misc'];
 const ignoreMethods = ['OPTIONS'];
 
 const getTrpcInput = (
-  request: FastifyRequest,
+  request: FastifyRequest
 ): Record<string, unknown> | undefined => {
   const input = path<any>(['query', 'input'], request);
   try {
@@ -18,7 +17,7 @@ const getTrpcInput = (
 
 export async function requestLoggingHook(
   request: FastifyRequest,
-  reply: FastifyReply,
+  reply: FastifyReply
 ) {
   if (ignoreMethods.includes(request.method)) {
     return;
@@ -40,9 +39,9 @@ export async function requestLoggingHook(
       elapsed: reply.elapsedTime,
       headers: pick(
         ['openpanel-client-id', 'openpanel-sdk-name', 'openpanel-sdk-version'],
-        request.headers,
+        request.headers
       ),
-      body: request.body,
+      // body: request.body,
     });
   }
 }

apps/api/src/index.ts

@@ -3,12 +3,12 @@ process.env.TZ = 'UTC';
 import compress from '@fastify/compress';
 import cookie from '@fastify/cookie';
 import cors, { type FastifyCorsOptions } from '@fastify/cors';
-import type { FastifyTRPCPluginOptions } from '@trpc/server/adapters/fastify';
-import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
-import type { FastifyBaseLogger, FastifyRequest } from 'fastify';
-import Fastify from 'fastify';
-import metricsPlugin from 'fastify-metrics';
-
+import {
+  decodeSessionToken,
+  EMPTY_SESSION,
+  type SessionValidationResult,
+  validateSessionToken,
+} from '@openpanel/auth';
 import { generateId } from '@openpanel/common';
 import {
   type IServiceClientWithProject,
@@ -17,13 +17,11 @@ import {
 import { getRedisPub } from '@openpanel/redis';
 import type { AppRouter } from '@openpanel/trpc';
 import { appRouter, createContext } from '@openpanel/trpc';
-
-import {
-  EMPTY_SESSION,
-  type SessionValidationResult,
-  decodeSessionToken,
-  validateSessionToken,
-} from '@openpanel/auth';
+import type { FastifyTRPCPluginOptions } from '@trpc/server/adapters/fastify';
+import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
+import type { FastifyBaseLogger, FastifyRequest } from 'fastify';
+import Fastify from 'fastify';
+import metricsPlugin from 'fastify-metrics';
 import sourceMapSupport from 'source-map-support';
 import {
   healthcheck,
@@ -72,7 +70,7 @@ const startServer = async () => {
   try {
     const fastify = Fastify({
       maxParamLength: 15_000,
-      bodyLimit: 1048576 * 500, // 500MB
+      bodyLimit: 1_048_576 * 500, // 500MB
       loggerInstance: logger as unknown as FastifyBaseLogger,
       disableRequestLogging: true,
       genReqId: (req) =>
@@ -84,7 +82,7 @@ const startServer = async () => {
     fastify.register(cors, () => {
       return (
         req: FastifyRequest,
-        callback: (error: Error | null, options: FastifyCorsOptions) => void,
+        callback: (error: Error | null, options: FastifyCorsOptions) => void
       ) => {
         // TODO: set prefix on dashboard routes
         const corsPaths = [
@@ -97,7 +95,7 @@ const startServer = async () => {
         ];
 
         const isPrivatePath = corsPaths.some((path) =>
-          req.url.startsWith(path),
+          req.url.startsWith(path)
         );
 
         if (isPrivatePath) {
@@ -118,6 +116,7 @@ const startServer = async () => {
 
         return callback(null, {
           origin: '*',
+          maxAge: 86_400 * 7, // cache preflight for 24h
         });
       };
     });
@@ -126,6 +125,11 @@ const startServer = async () => {
       global: false,
     });
 
+    fastify.addHook('onRequest', async (req) => {
+      if (req.method === 'POST') {
+        console.log('Incoming req', req.method, req.url);
+      }
+    });
     fastify.addHook('onRequest', requestIdHook);
     fastify.addHook('onRequest', timestampHook);
     fastify.addHook('onRequest', ipHook);
@@ -149,7 +153,7 @@ const startServer = async () => {
           try {
             const sessionId = decodeSessionToken(req.cookies?.session);
             const session = await runWithAlsSession(sessionId, () =>
-              validateSessionToken(req.cookies.session),
+              validateSessionToken(req.cookies.session)
             );
             req.session = session;
           } catch (e) {
@@ -158,7 +162,7 @@ const startServer = async () => {
         } else if (process.env.DEMO_USER_ID) {
           try {
             const session = await runWithAlsSession('1', () =>
-              validateSessionToken(null),
+              validateSessionToken(null)
             );
             req.session = session;
           } catch (e) {
@@ -173,7 +177,7 @@ const startServer = async () => {
         prefix: '/trpc',
         trpcOptions: {
           router: appRouter,
-          createContext: createContext,
+          createContext,
           onError(ctx) {
             if (
               ctx.error.code === 'UNAUTHORIZED' &&
@@ -217,7 +221,7 @@ const startServer = async () => {
         reply.send({
           status: 'ok',
           message: 'Successfully running OpenPanel.dev API',
-        }),
+        })
       );
     });
 
@@ -274,7 +278,7 @@ const startServer = async () => {
     } catch (error) {
       logger.warn('Failed to set redis notify-keyspace-events', error);
       logger.warn(
-        'If you use a managed Redis service, you may need to set this manually.',
+        'If you use a managed Redis service, you may need to set this manually.'
       );
       logger.warn('Otherwise some functions may not work as expected.');
     }