batching events

apps/api/src/controllers/event.controller.ts

@@ -5,6 +5,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
 import { generateDeviceId } from '@openpanel/common';
 import { getSalts } from '@openpanel/db';
 import { eventsQueue } from '@openpanel/queue';
+import { redis } from '@openpanel/redis';
 import type { PostEventPayload } from '@openpanel/sdk';
 
 export async function postEvent(
@@ -63,7 +64,6 @@ export async function postEvent(
   }
   const ip = getClientIp(request)!;
   const ua = request.headers['user-agent']!;
-  const origin = request.headers.origin!;
   const projectId = request.client?.projectId;
 
   if (!projectId) {
@@ -84,19 +84,15 @@ export async function postEvent(
     ip,
     ua,
   });
-  // TODO: Remove after 2024-09-26
-  const currentDeviceIdDeprecated = generateDeviceId({
-    salt: salts.current,
-    origin,
-    ip,
-    ua,
-  });
-  const previousDeviceIdDeprecated = generateDeviceId({
-    salt: salts.previous,
-    origin,
-    ip,
-    ua,
-  });
+
+  // this will ensure that we don't have multiple events creating sessions
+  const locked = await redis.set(
+    `request:priority:${currentDeviceId}-${previousDeviceId}`,
+    'locked',
+    'EX',
+    10,
+    'NX'
+  );
 
   eventsQueue.add('event', {
     type: 'incomingEvent',
@@ -105,13 +101,15 @@ export async function postEvent(
       headers: {
         ua,
       },
-      event: request.body,
+      event: {
+        ...request.body,
+        // Dont rely on the client for the timestamp
+        timestamp: new Date().toISOString(),
+      },
       geo,
       currentDeviceId,
       previousDeviceId,
-      // TODO: Remove after 2024-09-26
-      currentDeviceIdDeprecated,
-      previousDeviceIdDeprecated,
+      priority: locked === 'OK',
     },
   });
 

apps/api/src/controllers/live.controller.ts

@@ -19,7 +19,7 @@ export function getLiveEventInfo(key: string) {
   return key.split(':').slice(2) as [string, string];
 }
 
-export async function test(
+export async function testVisitors(
   req: FastifyRequest<{
     Params: {
       projectId: string;
@@ -28,13 +28,15 @@ export async function test(
   reply: FastifyReply
 ) {
   const events = await getEvents(
-    `SELECT * FROM events WHERE project_id = ${escape(req.params.projectId)} AND name = 'screen_view' LIMIT 500`
+    `SELECT * FROM events LIMIT 500`
+    // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
   );
   const event = events[Math.floor(Math.random() * events.length)];
   if (!event) {
     return reply.status(404).send('No event found');
   }
-  redisPub.publish('event', superjson.stringify(event));
+  event.projectId = req.params.projectId;
+  redisPub.publish('event:received', superjson.stringify(event));
   redis.set(
     `live:event:${event.projectId}:${Math.random() * 1000}`,
     '',
@@ -44,6 +46,26 @@ export async function test(
   reply.status(202).send(event);
 }
 
+export async function testEvents(
+  req: FastifyRequest<{
+    Params: {
+      projectId: string;
+    };
+  }>,
+  reply: FastifyReply
+) {
+  const events = await getEvents(
+    `SELECT * FROM events LIMIT 500`
+    // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
+  );
+  const event = events[Math.floor(Math.random() * events.length)];
+  if (!event) {
+    return reply.status(404).send('No event found');
+  }
+  redisPub.publish('event:saved', superjson.stringify(event));
+  reply.status(202).send(event);
+}
+
 export function wsVisitors(
   connection: {
     socket: WebSocket;
@@ -56,11 +78,11 @@ export function wsVisitors(
 ) {
   const { params } = req;
 
-  redisSub.subscribe('event');
+  redisSub.subscribe('event:received');
   redisSub.psubscribe('__key*:expired');
 
   const message = (channel: string, message: string) => {
-    if (channel === 'event') {
+    if (channel === 'event:received') {
       const event = getSuperJson<IServiceCreateEventPayload>(message);
       if (event?.projectId === params.projectId) {
         getLiveVisitors(params.projectId).then((count) => {
@@ -82,7 +104,7 @@ export function wsVisitors(
   redisSub.on('pmessage', pmessage);
 
   connection.socket.on('close', () => {
-    redisSub.unsubscribe('event');
+    redisSub.unsubscribe('event:saved');
     redisSub.punsubscribe('__key*:expired');
     redisSub.off('message', message);
     redisSub.off('pmessage', pmessage);
@@ -90,7 +112,7 @@ export function wsVisitors(
 }
 
 export function wsEvents(connection: { socket: WebSocket }) {
-  redisSub.subscribe('event');
+  redisSub.subscribe('event:saved');
 
   const message = (channel: string, message: string) => {
     const event = getSuperJson<IServiceCreateEventPayload>(message);
@@ -102,7 +124,7 @@ export function wsEvents(connection: { socket: WebSocket }) {
   redisSub.on('message', message);
 
   connection.socket.on('close', () => {
-    redisSub.unsubscribe('event');
+    redisSub.unsubscribe('event:saved');
     redisSub.off('message', message);
   });
 }
@@ -129,7 +151,7 @@ export async function wsProjectEvents(
     projectId: params.projectId,
   });
 
-  redisSub.subscribe('event');
+  redisSub.subscribe('event:saved');
 
   const message = async (channel: string, message: string) => {
     const event = getSuperJson<IServiceCreateEventPayload>(message);
@@ -151,7 +173,7 @@ export async function wsProjectEvents(
   redisSub.on('message', message as any);
 
   connection.socket.on('close', () => {
-    redisSub.unsubscribe('event');
+    redisSub.unsubscribe('event:saved');
     redisSub.off('message', message as any);
   });
 }

apps/api/src/controllers/profile.controller.ts

@@ -1,5 +1,5 @@
 import { getClientIp, parseIp } from '@/utils/parseIp';
-import { isUserAgentSet, parseUserAgent } from '@/utils/parseUserAgent';
+import { parseUserAgent } from '@/utils/parseUserAgent';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 import { assocPath, pathOr } from 'ramda';
 
@@ -29,7 +29,7 @@ export async function updateProfile(
     properties: {
       ...(properties ?? {}),
       ...(ip ? geo : {}),
-      ...(isUserAgentSet(ua) ? uaInfo : {}),
+      ...uaInfo,
     },
     ...rest,
   });

apps/api/src/controllers/webhook.controller.ts

@@ -0,0 +1,152 @@
+import type { WebhookEvent } from '@clerk/fastify';
+import type { FastifyReply, FastifyRequest } from 'fastify';
+import { pathOr } from 'ramda';
+import { Webhook } from 'svix';
+
+import { AccessLevel, db } from '@openpanel/db';
+
+if (!process.env.CLERK_SIGNING_SECRET) {
+  throw new Error('CLERK_SIGNING_SECRET is required');
+}
+
+const wh = new Webhook(process.env.CLERK_SIGNING_SECRET);
+
+function verify(body: any, headers: FastifyRequest['headers']) {
+  try {
+    const svix_id = headers['svix-id'] as string;
+    const svix_timestamp = headers['svix-timestamp'] as string;
+    const svix_signature = headers['svix-signature'] as string;
+
+    wh.verify(JSON.stringify(body), {
+      'svix-id': svix_id,
+      'svix-timestamp': svix_timestamp,
+      'svix-signature': svix_signature,
+    });
+
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+export async function clerkWebhook(
+  request: FastifyRequest<{
+    Body: WebhookEvent;
+  }>,
+  reply: FastifyReply
+) {
+  const payload = request.body;
+  const verified = verify(payload, request.headers);
+
+  if (!verified) {
+    return reply.send({ message: 'Invalid signature' });
+  }
+
+  if (payload.type === 'user.created') {
+    const email = payload.data.email_addresses[0]?.email_address;
+
+    if (!email) {
+      return Response.json(
+        { message: 'No email address found' },
+        { status: 400 }
+      );
+    }
+
+    const user = await db.user.create({
+      data: {
+        id: payload.data.id,
+        email,
+        firstName: payload.data.first_name,
+        lastName: payload.data.last_name,
+      },
+    });
+
+    const memberships = await db.member.findMany({
+      where: {
+        email,
+        userId: null,
+      },
+    });
+
+    for (const membership of memberships) {
+      const access = pathOr<string[]>([], ['meta', 'access'], membership);
+      db.$transaction([
+        // Update the member to link it to the user
+        // This will remove the item from invitations
+        db.member.update({
+          where: {
+            id: membership.id,
+          },
+          data: {
+            userId: user.id,
+          },
+        }),
+        db.projectAccess.createMany({
+          data: access
+            .filter((a) => typeof a === 'string')
+            .map((projectId) => ({
+              organizationSlug: membership.organizationId,
+              organizationId: membership.organizationId,
+              projectId: projectId,
+              userId: user.id,
+              level: AccessLevel.read,
+            })),
+        }),
+      ]);
+    }
+  }
+
+  if (payload.type === 'organizationMembership.created') {
+    const access = payload.data.public_metadata.access;
+    if (Array.isArray(access)) {
+      await db.projectAccess.createMany({
+        data: access
+          .filter((a): a is string => typeof a === 'string')
+          .map((projectId) => ({
+            organizationSlug: payload.data.organization.slug,
+            organizationId: payload.data.organization.slug,
+            projectId: projectId,
+            userId: payload.data.public_user_data.user_id,
+            level: AccessLevel.read,
+          })),
+      });
+    }
+  }
+
+  if (payload.type === 'user.deleted') {
+    await db.$transaction([
+      db.user.update({
+        where: {
+          id: payload.data.id,
+        },
+        data: {
+          deletedAt: new Date(),
+          firstName: null,
+          lastName: null,
+          email: `deleted+${payload.data.id}@openpanel.dev`,
+        },
+      }),
+      db.projectAccess.deleteMany({
+        where: {
+          userId: payload.data.id,
+        },
+      }),
+      db.member.deleteMany({
+        where: {
+          userId: payload.data.id,
+        },
+      }),
+    ]);
+  }
+
+  if (payload.type === 'organizationMembership.deleted') {
+    await db.projectAccess.deleteMany({
+      where: {
+        organizationSlug: payload.data.organization.slug,
+        userId: payload.data.public_user_data.user_id,
+      },
+    });
+  }
+
+  reply.send({ success: true });
+}

apps/api/src/index.ts

@@ -19,6 +19,7 @@ import exportRouter from './routes/export.router';
 import liveRouter from './routes/live.router';
 import miscRouter from './routes/misc.router';
 import profileRouter from './routes/profile.router';
+import webhookRouter from './routes/webhook.router';
 import { logger, logInfo } from './utils/logger';
 
 declare module 'fastify' {
@@ -82,6 +83,7 @@ const startServer = async () => {
     fastify.register(profileRouter, { prefix: '/profile' });
     fastify.register(miscRouter, { prefix: '/misc' });
     fastify.register(exportRouter, { prefix: '/export' });
+    fastify.register(webhookRouter, { prefix: '/webhook' });
     fastify.setErrorHandler((error) => {
       logger.error(error, 'Error in request');
     });
@@ -89,35 +91,45 @@ const startServer = async () => {
       reply.send({ name: 'openpanel sdk api' });
     });
     fastify.get('/healthcheck', async (request, reply) => {
-      try {
-        const redisRes = await withTimings(redis.keys('*'));
-        const dbRes = await withTimings(db.project.findFirst());
-        const queueRes = await withTimings(eventsQueue.getCompleted());
-        const chRes = await withTimings(
-          chQuery('SELECT * FROM events LIMIT 1')
-        );
+      const redisRes = await withTimings(redis.keys('*')).catch(
+        (e: Error) => e
+      );
+      const dbRes = await withTimings(db.project.findFirst()).catch(
+        (e: Error) => e
+      );
+      const queueRes = await withTimings(eventsQueue.getCompleted()).catch(
+        (e: Error) => e
+      );
+      const chRes = await withTimings(
+        chQuery('SELECT * FROM events LIMIT 1')
+      ).catch((e: Error) => e);
 
-        reply.send({
-          redis: {
-            ok: redisRes[1].length ? true : false,
-            time: `${redisRes[0]}ms`,
-          },
-          db: {
-            ok: dbRes[1] ? true : false,
-            time: `${dbRes[0]}ms`,
-          },
-          queue: {
-            ok: queueRes[1].length ? true : false,
-            time: `${queueRes[0]}ms`,
-          },
-          ch: {
-            ok: chRes[1].length ? true : false,
-            time: `${chRes[0]}ms`,
-          },
-        });
-      } catch (e) {
-        reply.status(500).send();
-      }
+      reply.send({
+        redis: Array.isArray(redisRes)
+          ? {
+              ok: redisRes[1].length ? true : false,
+              time: `${redisRes[0]}ms`,
+            }
+          : redisRes,
+        db: Array.isArray(dbRes)
+          ? {
+              ok: dbRes[1] ? true : false,
+              time: `${dbRes[0]}ms`,
+            }
+          : dbRes,
+        queue: Array.isArray(queueRes)
+          ? {
+              ok: queueRes[1].length ? true : false,
+              time: `${queueRes[0]}ms`,
+            }
+          : queueRes,
+        ch: Array.isArray(chRes)
+          ? {
+              ok: chRes[1].length ? true : false,
+              time: `${chRes[0]}ms`,
+            }
+          : chRes,
+      });
     });
     if (process.env.NODE_ENV === 'production') {
       for (const signal of ['SIGINT', 'SIGTERM']) {

apps/api/src/routes/live.router.ts

@@ -3,10 +3,15 @@ import fastifyWS from '@fastify/websocket';
 import type { FastifyPluginCallback } from 'fastify';
 
 const liveRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.route({
+    method: 'GET',
+    url: '/visitors/test/:projectId',
+    handler: controller.testVisitors,
+  });
   fastify.route({
     method: 'GET',
     url: '/events/test/:projectId',
-    handler: controller.test,
+    handler: controller.testEvents,
   });
 
   fastify.register(fastifyWS);

apps/api/src/routes/webhook.router.ts

@@ -0,0 +1,13 @@
+import * as controller from '@/controllers/webhook.controller';
+import type { FastifyPluginCallback } from 'fastify';
+
+const webhookRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.route({
+    method: 'POST',
+    url: '/clerk',
+    handler: controller.clerkWebhook,
+  });
+  done();
+};
+
+export default webhookRouter;

apps/api/src/utils/auth.ts

@@ -5,8 +5,6 @@ import { verifyPassword } from '@openpanel/common';
 import type { Client, IServiceClient } from '@openpanel/db';
 import { ClientType, db } from '@openpanel/db';
 
-import { logger } from './logger';
-
 const cleanDomain = (domain: string) =>
   domain
     .replace('www.', '')

apps/api/src/utils/parseUserAgent.ts

@@ -1,11 +1,16 @@
 import { UAParser } from 'ua-parser-js';
 
-export function isUserAgentSet(ua: string) {
-  return ua !== 'node' && ua !== 'undici' && !!ua;
-}
+const parsedServerUa = {
+  isServer: true,
+  device: 'server',
+} as const;
 
-export function parseUserAgent(ua: string) {
+export function parseUserAgent(ua?: string | null) {
+  if (!ua) return parsedServerUa;
   const res = new UAParser(ua).getResult();
+
+  if (isServer(ua)) return parsedServerUa;
+
   return {
     os: res.os.name,
     osVersion: res.os.version,
@@ -14,20 +19,86 @@ export function parseUserAgent(ua: string) {
     device: res.device.type ?? getDevice(ua),
     brand: res.device.vendor,
     model: res.device.model,
-  };
+    isServer: false,
+  } as const;
+}
+
+const userAgentServerList: string[] = [
+  // Node.js libraries
+  'node',
+  'node-fetch',
+  'axios',
+  'request',
+  'superagent',
+  'undici',
+
+  // Python libraries
+  'python-requests',
+  'python-urllib',
+
+  // Ruby libraries
+  'Faraday',
+  'Ruby',
+  'http.rb',
+
+  // Go libraries
+  'Go-http-client',
+  'Go-http-client',
+
+  // Java libraries
+  'Apache-HttpClient',
+  'okhttp',
+  'okhowtp',
+
+  // PHP libraries
+  'GuzzleHttp',
+  'PHP-cURL',
+
+  // Other
+  'Dart',
+  'RestSharp', // Popular .NET HTTP client library
+  'HttpClientFactory', // .NET's typed client factory
+  'Ktor', // A client for Kotlin
+  'Ning', // Async HTTP client for Java
+  'grpc-csharp', // gRPC for C#
+  'Volley', // HTTP library used in Android apps for making network requests
+  'Spring',
+  'vert.x',
+  'grpc-',
+];
+
+function isServer(userAgent: string) {
+  const match = userAgentServerList.some((server) =>
+    userAgent.includes(server)
+  );
+  if (match) {
+    return true;
+  }
+
+  return !!userAgent.match(/^[^\s]+\/[\d.]+$/);
 }
 
 export function getDevice(ua: string) {
-  const t1 =
+  const mobile1 =
     /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(
       ua
     );
-  const t2 =
+  const mobile2 =
     /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw-(n|u)|c55\/|capi|ccwa|cdm-|cell|chtm|cldc|cmd-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc-s|devi|dica|dmob|do(c|p)o|ds(12|-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(-|_)|g1 u|g560|gene|gf-5|g-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd-(m|p|t)|hei-|hi(pt|ta)|hp( i|ip)|hs-c|ht(c(-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i-(20|go|ma)|i230|iac( |-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|-[a-w])|libw|lynx|m1-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|-([1-8]|c))|phil|pire|pl(ay|uc)|pn-2|po(ck|rt|se)|prox|psio|pt-g|qa-a|qc(07|12|21|32|60|-[2-7]|i-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h-|oo|p-)|sdk\/|se(c(-|0|1)|47|mc|nd|ri)|sgh-|shar|sie(-|m)|sk-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h-|v-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl-|tdg-|tel(i|m)|tim-|t-mo|to(pl|sh)|ts(70|m-|m3|m5)|tx-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas-|your|zeto|zte-/i.test(
       ua.slice(0, 4)
     );
-  if (t1 || t2) {
+  const tablet =
+    /tablet|ipad|android(?!.*mobile)|xoom|sch-i800|kindle|silk|playbook/i.test(
+      ua
+    );
+
+  if (mobile1 || mobile2) {
     return 'mobile';
   }
+
+  if (tablet) {
+    return 'tablet';
+  }
+
   return 'desktop';
 }