batching events

apps/api/src/controllers/event.controller.ts

@@ -5,6 +5,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
 import { generateDeviceId } from '@openpanel/common';
 import { getSalts } from '@openpanel/db';
 import { eventsQueue } from '@openpanel/queue';
+import { redis } from '@openpanel/redis';
 import type { PostEventPayload } from '@openpanel/sdk';
 
 export async function postEvent(
@@ -63,7 +64,6 @@ export async function postEvent(
   }
   const ip = getClientIp(request)!;
   const ua = request.headers['user-agent']!;
-  const origin = request.headers.origin!;
   const projectId = request.client?.projectId;
 
   if (!projectId) {
@@ -84,19 +84,15 @@ export async function postEvent(
     ip,
     ua,
   });
-  // TODO: Remove after 2024-09-26
-  const currentDeviceIdDeprecated = generateDeviceId({
-    salt: salts.current,
-    origin,
-    ip,
-    ua,
-  });
-  const previousDeviceIdDeprecated = generateDeviceId({
-    salt: salts.previous,
-    origin,
-    ip,
-    ua,
-  });
+
+  // this will ensure that we don't have multiple events creating sessions
+  const locked = await redis.set(
+    `request:priority:${currentDeviceId}-${previousDeviceId}`,
+    'locked',
+    'EX',
+    10,
+    'NX'
+  );
 
   eventsQueue.add('event', {
     type: 'incomingEvent',
@@ -105,13 +101,15 @@ export async function postEvent(
       headers: {
         ua,
       },
-      event: request.body,
+      event: {
+        ...request.body,
+        // Dont rely on the client for the timestamp
+        timestamp: new Date().toISOString(),
+      },
       geo,
       currentDeviceId,
       previousDeviceId,
-      // TODO: Remove after 2024-09-26
-      currentDeviceIdDeprecated,
-      previousDeviceIdDeprecated,
+      priority: locked === 'OK',
     },
   });
 

apps/api/src/controllers/live.controller.ts

@@ -19,7 +19,7 @@ export function getLiveEventInfo(key: string) {
   return key.split(':').slice(2) as [string, string];
 }
 
-export async function test(
+export async function testVisitors(
   req: FastifyRequest<{
     Params: {
       projectId: string;
@@ -28,13 +28,15 @@ export async function test(
   reply: FastifyReply
 ) {
   const events = await getEvents(
-    `SELECT * FROM events WHERE project_id = ${escape(req.params.projectId)} AND name = 'screen_view' LIMIT 500`
+    `SELECT * FROM events LIMIT 500`
+    // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
   );
   const event = events[Math.floor(Math.random() * events.length)];
   if (!event) {
     return reply.status(404).send('No event found');
   }
-  redisPub.publish('event', superjson.stringify(event));
+  event.projectId = req.params.projectId;
+  redisPub.publish('event:received', superjson.stringify(event));
   redis.set(
     `live:event:${event.projectId}:${Math.random() * 1000}`,
     '',
@@ -44,6 +46,26 @@ export async function test(
   reply.status(202).send(event);
 }
 
+export async function testEvents(
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>,
+  reply: FastifyReply
+) {
+  const events = await getEvents(
+    `SELECT * FROM events LIMIT 500`
+    // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
+  );
+  const event = events[Math.floor(Math.random() * events.length)];
+  if (!event) {
+    return reply.status(404).send('No event found');
+  }
+  redisPub.publish('event:saved', superjson.stringify(event));
+  reply.status(202).send(event);
+}
+
 export function wsVisitors(
   connection: {
     socket: WebSocket;
@@ -56,11 +78,11 @@ export function wsVisitors(
 ) {
   const { params } = req;
 
-  redisSub.subscribe('event');
+  redisSub.subscribe('event:received');
   redisSub.psubscribe('__key*:expired');
 
   const message = (channel: string, message: string) => {
-    if (channel === 'event') {
+    if (channel === 'event:received') {
       const event = getSuperJson<IServiceCreateEventPayload>(message);
       if (event?.projectId === params.projectId) {
         getLiveVisitors(params.projectId).then((count) => {
@@ -82,7 +104,7 @@ export function wsVisitors(
   redisSub.on('pmessage', pmessage);
 
   connection.socket.on('close', () => {
-    redisSub.unsubscribe('event');
+    redisSub.unsubscribe('event:saved');
     redisSub.punsubscribe('__key*:expired');
     redisSub.off('message', message);
     redisSub.off('pmessage', pmessage);
@@ -90,7 +112,7 @@ export function wsVisitors(
 }
 
 export function wsEvents(connection: { socket: WebSocket }) {
-  redisSub.subscribe('event');
+  redisSub.subscribe('event:saved');
 
   const message = (channel: string, message: string) => {
     const event = getSuperJson<IServiceCreateEventPayload>(message);
@@ -102,7 +124,7 @@ export function wsEvents(connection: { socket: WebSocket }) {
   redisSub.on('message', message);
 
   connection.socket.on('close', () => {
-    redisSub.unsubscribe('event');
+    redisSub.unsubscribe('event:saved');
     redisSub.off('message', message);
   });
 }
@@ -129,7 +151,7 @@ export async function wsProjectEvents(
     projectId: params.projectId,
   });
 
-  redisSub.subscribe('event');
+  redisSub.subscribe('event:saved');
 
   const message = async (channel: string, message: string) => {
     const event = getSuperJson<IServiceCreateEventPayload>(message);
@@ -151,7 +173,7 @@ export async function wsProjectEvents(
   redisSub.on('message', message as any);
 
   connection.socket.on('close', () => {
-    redisSub.unsubscribe('event');
+    redisSub.unsubscribe('event:saved');
     redisSub.off('message', message as any);
   });
 }

apps/api/src/controllers/profile.controller.ts

@@ -1,5 +1,5 @@
 import { getClientIp, parseIp } from '@/utils/parseIp';
-import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
+import { parseUserAgent } from '@/utils/parseUserAgent';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { assocPath, pathOr } from 'ramda';
 
@@ -29,7 +29,7 @@ export async function updateProfile(
     properties: {
       ...(properties ?? {}),
       ...(ip ? geo : {}),
-      ...(isUserAgentSet(ua) ? uaInfo : {}),
+      ...uaInfo,
     },
     ...rest,
   });

apps/api/src/controllers/webhook.controller.ts

@@ -0,0 +1,152 @@
+import type { WebhookEvent } from '@clerk/fastify';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { pathOr } from 'ramda';
+import { Webhook } from 'svix';
+
+import { AccessLevel, db } from '@openpanel/db';
+
+if (!process.env.CLERK_SIGNING_SECRET) {
+  throw new Error('CLERK_SIGNING_SECRET is required');
+}
+
+const wh = new Webhook(process.env.CLERK_SIGNING_SECRET);
+
+function verify(body: any, headers: FastifyRequest['headers']) {
+  try {
+    const svix_id = headers['svix-id'] as string;
+    const svix_timestamp = headers['svix-timestamp'] as string;
+    const svix_signature = headers['svix-signature'] as string;
+
+    wh.verify(JSON.stringify(body), {
+      'svix-id': svix_id,
+      'svix-timestamp': svix_timestamp,
+      'svix-signature': svix_signature,
+    });
+
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+export async function clerkWebhook(
+  request: FastifyRequest<{
+    Body: WebhookEvent;
+  }>,
+  reply: FastifyReply
+) {
+  const payload = request.body;
+  const verified = verify(payload, request.headers);
+
+  if (!verified) {
+    return reply.send({ message: 'Invalid signature' });
+  }
+
+  if (payload.type === 'user.created') {
+    const email = payload.data.email_addresses[0]?.email_address;
+
+    if (!email) {
+      return Response.json(
+        { message: 'No email address found' },
+        { status: 400 }
+      );
+    }
+
+    const user = await db.user.create({
+      data: {
+        id: payload.data.id,
+        email,
+        firstName: payload.data.first_name,
+        lastName: payload.data.last_name,
+      },
+    });
+
+    const memberships = await db.member.findMany({
+      where: {
+        email,
+        userId: null,
+      },
+    });
+
+    for (const membership of memberships) {
+      const access = pathOr<string[]>([], ['meta', 'access'], membership);
+      db.$transaction([
+        // Update the member to link it to the user
+        // This will remove the item from invitations
+        db.member.update({
+          where: {
+            id: membership.id,
+          },
+          data: {
+            userId: user.id,
+          },
+        }),
+        db.projectAccess.createMany({
+          data: access
+            .filter((a) => typeof a === 'string')
+            .map((projectId) => ({
+              organizationSlug: membership.organizationId,
+              organizationId: membership.organizationId,
+              projectId: projectId,
+              userId: user.id,
+              level: AccessLevel.read,
+            })),
+        }),
+      ]);
+    }
+  }
+
+  if (payload.type === 'organizationMembership.created') {
+    const access = payload.data.public_metadata.access;
+    if (Array.isArray(access)) {
+      await db.projectAccess.createMany({
+        data: access
+          .filter((a): a is string => typeof a === 'string')
+          .map((projectId) => ({
+            organizationSlug: payload.data.organization.slug,
+            organizationId: payload.data.organization.slug,
+            projectId: projectId,
+            userId: payload.data.public_user_data.user_id,
+            level: AccessLevel.read,
+          })),
+      });
+    }
+  }
+
+  if (payload.type === 'user.deleted') {
+    await db.$transaction([
+      db.user.update({
+        where: {
+          id: payload.data.id,
+        },
+        data: {
+          deletedAt: new Date(),
+          firstName: null,
+          lastName: null,
+          email: `deleted+${payload.data.id}@openpanel.dev`,
+        },
+      }),
+      db.projectAccess.deleteMany({
+        where: {
+          userId: payload.data.id,
+        },
+      }),
+      db.member.deleteMany({
+        where: {
+          userId: payload.data.id,
+        },
+      }),
+    ]);
+  }
+
+  if (payload.type === 'organizationMembership.deleted') {
+    await db.projectAccess.deleteMany({
+      where: {
+        organizationSlug: payload.data.organization.slug,
+        userId: payload.data.public_user_data.user_id,
+      },
+    });
+  }
+
+  reply.send({ success: true });
+}