feature(dashboard): add ability to filter out events by profile id and ip (#101)

apps/api/scripts/migrate-client-settings.ts

@@ -0,0 +1,124 @@
+import { stripTrailingSlash } from '@openpanel/common';
+import {
+  chQuery,
+  db,
+  getClientByIdCached,
+  getProjectByIdCached,
+} from '@openpanel/db';
+
+const pickBestDomain = (domains: string[]): string | null => {
+  // Filter out invalid domains
+  const validDomains = domains.filter(
+    (domain) =>
+      domain &&
+      !domain.includes('*') &&
+      !domain.includes('localhost') &&
+      !domain.includes('127.0.0.1'),
+  );
+
+  if (validDomains.length === 0) return null;
+
+  // Score each domain
+  const scoredDomains = validDomains.map((domain) => {
+    let score = 0;
+
+    // Prefer https (highest priority)
+    if (domain.startsWith('https://')) score += 100;
+
+    // Penalize domains from common providers like vercel, netlify, etc.
+    if (
+      domain.includes('vercel.app') ||
+      domain.includes('netlify.app') ||
+      domain.includes('herokuapp.com') ||
+      domain.includes('github.io') ||
+      domain.includes('gitlab.io') ||
+      domain.includes('surge.sh') ||
+      domain.includes('cloudfront.net') ||
+      domain.includes('firebaseapp.com') ||
+      domain.includes('azurestaticapps.net') ||
+      domain.includes('pages.dev') ||
+      domain.includes('ngrok-free.app') ||
+      domain.includes('ngrok.app')
+    ) {
+      score -= 50;
+    }
+
+    // Penalize subdomains
+    const domainParts = domain
+      .replace('https://', '')
+      .replace('http://', '')
+      .split('.');
+    if (domainParts.length <= 2) score += 50;
+
+    // Tiebreaker: prefer shorter domains
+    score -= domain.length;
+
+    return { domain, score };
+  });
+
+  // Sort by score (highest first) and return the best domain
+  const bestDomain = scoredDomains.sort((a, b) => b.score - a.score)[0];
+  return bestDomain?.domain || null;
+};
+
+async function main() {
+  const projects = await db.project.findMany({
+    include: {
+      clients: true,
+    },
+  });
+
+  const matches = [];
+  for (const project of projects) {
+    const cors = [];
+    let crossDomain = false;
+    for (const client of project.clients) {
+      if (client.crossDomain) {
+        crossDomain = true;
+      }
+      cors.push(
+        ...(client.cors?.split(',') ?? []).map((c) =>
+          stripTrailingSlash(c.trim()),
+        ),
+      );
+      await getClientByIdCached.clear(client.id);
+    }
+
+    let domain = pickBestDomain(cors);
+
+    if (!domain) {
+      const res = await chQuery<{ origin: string }>(
+        `SELECT origin FROM events_distributed WHERE project_id = '${project.id}' and origin != ''`,
+      );
+      if (res.length) {
+        domain = pickBestDomain(res.map((r) => r.origin));
+        matches.push(domain);
+      } else {
+        console.log('No domain found for client');
+      }
+    }
+
+    await db.project.update({
+      where: { id: project.id },
+      data: {
+        cors,
+        crossDomain,
+        domain,
+      },
+    });
+    console.log('Updated', {
+      cors,
+      crossDomain,
+      domain,
+    });
+
+    await getProjectByIdCached.clear(project.id);
+  }
+
+  console.log('DONE');
+  console.log('DONE');
+  console.log('DONE');
+  console.log('DONE');
+}
+
+main();

apps/api/src/controllers/event.controller.ts

@@ -54,7 +54,7 @@ export async function postEvent(
     {
       type: 'incomingEvent',
       payload: {
-        projectId: request.projectId,
+        projectId,
         headers: getStringHeaders(request.headers),
         event: {
           ...request.body,

apps/api/src/controllers/import.controller.ts

@@ -10,12 +10,17 @@ export async function importEvents(
   }>,
   reply: FastifyReply,
 ) {
+  const projectId = request.client?.projectId;
+  if (!projectId) {
+    throw new Error('Project ID is required');
+  }
+
   const importedAt = formatClickhouseDate(new Date());
   const values: IClickhouseEvent[] = request.body.map((event) => {
     return {
       ...event,
       properties: toDots(event.properties),
-      project_id: request.client?.projectId ?? '',
+      project_id: projectId,
       created_at: formatClickhouseDate(event.created_at),
       imported_at: importedAt,
     };

apps/api/src/controllers/profile.controller.ts

@@ -16,7 +16,10 @@ export async function updateProfile(
   reply: FastifyReply,
 ) {
   const { profileId, properties, ...rest } = request.body;
-  const projectId = request.projectId;
+  const projectId = request.client!.projectId;
+  if (!projectId) {
+    return reply.status(400).send('No projectId');
+  }
   const ip = getClientIp(request)!;
   const ua = request.headers['user-agent']!;
   const uaInfo = parseUserAgent(ua, properties);
@@ -44,7 +47,10 @@ export async function incrementProfileProperty(
   reply: FastifyReply,
 ) {
   const { profileId, property, value } = request.body;
-  const projectId = request.projectId;
+  const projectId = request.client!.projectId;
+  if (!projectId) {
+    return reply.status(400).send('No projectId');
+  }
 
   const profile = await getProfileById(profileId, projectId);
   if (!profile) {
@@ -83,7 +89,10 @@ export async function decrementProfileProperty(
   reply: FastifyReply,
 ) {
   const { profileId, property, value } = request.body;
-  const projectId = request.projectId;
+  const projectId = request.client?.projectId;
+  if (!projectId) {
+    return reply.status(400).send('No projectId');
+  }
 
   const profile = await getProfileById(profileId, projectId);
   if (!profile) {

apps/api/src/hooks/client.hook.ts

@@ -1,20 +1,9 @@
 import { SdkAuthError, validateSdkRequest } from '@/utils/auth';
-import type { TrackHandlerPayload } from '@openpanel/sdk';
-import type {
-  FastifyReply,
-  FastifyRequest,
-  HookHandlerDoneFunction,
-} from 'fastify';
+import type { FastifyReply, FastifyRequest } from 'fastify';
 
-export async function clientHook(
-  req: FastifyRequest<{
-    Body: TrackHandlerPayload;
-  }>,
-  reply: FastifyReply,
-) {
+export async function clientHook(req: FastifyRequest, reply: FastifyReply) {
   try {
-    const client = await validateSdkRequest(req.headers);
-    req.projectId = client.projectId;
+    const client = await validateSdkRequest(req);
     req.client = client;
   } catch (error) {
     if (error instanceof SdkAuthError) {

apps/api/src/hooks/ip.hook.ts

@@ -0,0 +1,18 @@
+import { getClientIp } from '@/utils/parseIp';
+import type {
+  FastifyReply,
+  FastifyRequest,
+  HookHandlerDoneFunction,
+} from 'fastify';
+
+export function ipHook(
+  request: FastifyRequest,
+  reply: FastifyReply,
+  done: HookHandlerDoneFunction,
+) {
+  const ip = getClientIp(request);
+  if (ip) {
+    request.clientIp = ip;
+  }
+  done();
+}

apps/api/src/index.ts

@@ -11,7 +11,7 @@ import metricsPlugin from 'fastify-metrics';
 import { path, pick } from 'ramda';
 
 import { generateId } from '@openpanel/common';
-import type { IServiceClient } from '@openpanel/db';
+import type { IServiceClient, IServiceClientWithProject } from '@openpanel/db';
 import { getRedisPub } from '@openpanel/redis';
 import type { AppRouter } from '@openpanel/trpc';
 import { appRouter, createContext } from '@openpanel/trpc';
@@ -21,6 +21,7 @@ import {
   healthcheck,
   healthcheckQueue,
 } from './controllers/healthcheck.controller';
+import { ipHook } from './hooks/ip.hook';
 import { requestIdHook } from './hooks/request-id.hook';
 import { requestLoggingHook } from './hooks/request-logging.hook';
 import { timestampHook } from './hooks/timestamp.hook';
@@ -38,8 +39,8 @@ sourceMapSupport.install();
 
 declare module 'fastify' {
   interface FastifyRequest {
-    projectId: string;
-    client: IServiceClient | null;
+    client: IServiceClientWithProject | null;
+    clientIp?: string;
     timestamp?: number;
   }
 }
@@ -60,6 +61,7 @@ const startServer = async () => {
           : generateId(),
     });
 
+    fastify.addHook('preHandler', ipHook);
     fastify.addHook('preHandler', timestampHook);
     fastify.addHook('onRequest', requestIdHook);
     fastify.addHook('onResponse', requestLoggingHook);

apps/api/src/utils/auth.ts

@@ -1,9 +1,19 @@
-import type { RawRequestDefaultExpression } from 'fastify';
+import type { FastifyRequest, RawRequestDefaultExpression } from 'fastify';
 import jwt from 'jsonwebtoken';
 
 import { verifyPassword } from '@openpanel/common/server';
-import type { Client, IServiceClient } from '@openpanel/db';
-import { ClientType, db } from '@openpanel/db';
+import type {
+  Client,
+  IServiceClient,
+  IServiceClientWithProject,
+} from '@openpanel/db';
+import { ClientType, db, getClientByIdCached } from '@openpanel/db';
+import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
+import type {
+  IProjectFilterIp,
+  IProjectFilterProfileId,
+} from '@openpanel/validation';
+import { path } from 'ramda';
 
 const cleanDomain = (domain: string) =>
   domain
@@ -32,11 +42,12 @@ export class SdkAuthError extends Error {
   }
 }
 
-type ClientWithProjectId = Client & { projectId: string };
-
 export async function validateSdkRequest(
-  headers: RawRequestDefaultExpression['headers'],
-): Promise<ClientWithProjectId> {
+  req: FastifyRequest<{
+    Body: PostEventPayload | TrackHandlerPayload;
+  }>,
+): Promise<IServiceClientWithProject> {
+  const { headers, clientIp } = req;
   const clientIdNew = headers['openpanel-client-id'] as string;
   const clientIdOld = headers['mixan-client-id'] as string;
   const clientSecretNew = headers['openpanel-client-secret'] as string;
@@ -59,22 +70,38 @@ export async function validateSdkRequest(
     throw createError('Ingestion: Missing client id');
   }
 
-  const client = await db.client.findUnique({
-    where: {
-      id: clientId,
-    },
-  });
+  const client = await getClientByIdCached(clientId);
 
   if (!client) {
     throw createError('Ingestion: Invalid client id');
   }
 
-  if (!client.projectId) {
+  if (!client.project) {
     throw createError('Ingestion: Client has no project');
   }
 
-  if (client.cors) {
-    const domainAllowed = client.cors.split(',').find((domain) => {
+  // Filter out blocked IPs
+  const ipFilter = client.project.filters.filter(
+    (filter): filter is IProjectFilterIp => filter.type === 'ip',
+  );
+  if (ipFilter.some((filter) => filter.ip === clientIp)) {
+    throw createError('Ingestion: IP address is blocked by project filter');
+  }
+
+  // Filter out blocked profile ids
+  const profileFilter = client.project.filters.filter(
+    (filter): filter is IProjectFilterProfileId => filter.type === 'profile_id',
+  );
+  const profileId =
+    path<string | undefined>(['payload', 'profileId'], req.body) || // Track handler
+    path<string | undefined>(['profileId'], req.body); // Event handler
+
+  if (profileFilter.some((filter) => filter.profileId === profileId)) {
+    throw createError('Ingestion: Profile id is blocked by project filter');
+  }
+
+  if (client.project.cors) {
+    const domainAllowed = client.project.cors.find((domain) => {
       const cleanedDomain = cleanDomain(domain);
       // support wildcard domains `*.foo.com`
       if (cleanedDomain.includes('*')) {
@@ -91,17 +118,17 @@ export async function validateSdkRequest(
     });
 
     if (domainAllowed) {
-      return client as ClientWithProjectId;
+      return client;
     }
 
-    if (client.cors === '*' && origin) {
-      return client as ClientWithProjectId;
+    if (client.project.cors.includes('*') && origin) {
+      return client;
     }
   }
 
   if (client.secret && clientSecret) {
     if (await verifyPassword(clientSecret, client.secret)) {
-      return client as ClientWithProjectId;
+      return client;
     }
   }
 
@@ -110,14 +137,10 @@ export async function validateSdkRequest(
 
 export async function validateExportRequest(
   headers: RawRequestDefaultExpression['headers'],
-): Promise<IServiceClient> {
+): Promise<IServiceClientWithProject> {
   const clientId = headers['openpanel-client-id'] as string;
   const clientSecret = (headers['openpanel-client-secret'] as string) || '';
-  const client = await db.client.findUnique({
-    where: {
-      id: clientId,
-    },
-  });
+  const client = await getClientByIdCached(clientId);
 
   if (!client) {
     throw new Error('Export: Invalid client id');
@@ -140,14 +163,10 @@ export async function validateExportRequest(
 
 export async function validateImportRequest(
   headers: RawRequestDefaultExpression['headers'],
-): Promise<IServiceClient> {
+): Promise<IServiceClientWithProject> {
   const clientId = headers['openpanel-client-id'] as string;
   const clientSecret = (headers['openpanel-client-secret'] as string) || '';
-  const client = await db.client.findUnique({
-    where: {
-      id: clientId,
-    },
-  });
+  const client = await getClientByIdCached(clientId);
 
   if (!client) {
     throw new Error('Import: Invalid client id');

apps/api/src/utils/rate-limiter.ts

@@ -28,7 +28,9 @@ export async function activateRateLimiter({
         req.headers['x-forwarded-for']) as string;
     },
     onExceeded: (req, reply) => {
-      req.log.warn('Rate limit exceeded');
+      req.log.warn('Rate limit exceeded', {
+        clientId: req.headers['openpanel-client-id'],
+      });
     },
   });
 }