api: add first version of export api

apps/api/src/controllers/export.controller.ts

@@ -0,0 +1,88 @@
+import type { FastifyReply, FastifyRequest } from 'fastify';
+
+import type { GetEventListOptions } from '@openpanel/db';
+import { ClientType, db, getEventList, getEventsCount } from '@openpanel/db';
+
+type EventsQuery = {
+  project_id?: string;
+  event?: string | string[];
+  start?: string;
+  end?: string;
+  page?: string;
+};
+export async function events(
+  request: FastifyRequest<{
+    Querystring: EventsQuery;
+  }>,
+  reply: FastifyReply
+) {
+  const query = request.query;
+
+  if (query.project_id) {
+    if (
+      request.client?.type === ClientType.read &&
+      request.client?.projectId !== query.project_id
+    ) {
+      reply.status(403).send({
+        error: 'Forbidden',
+        message: 'You do not have access to this project',
+      });
+      return;
+    }
+
+    const project = await db.project.findUnique({
+      where: {
+        organizationSlug: request.client?.organizationSlug,
+        id: query.project_id,
+      },
+    });
+
+    if (!project) {
+      reply.status(404).send({
+        error: 'Not Found',
+        message: 'Project not found',
+      });
+      return;
+    }
+  }
+
+  const projectId = query.project_id ?? request.client?.projectId;
+
+  if (!projectId) {
+    reply.status(400).send({
+      error: 'Bad Request',
+      message: 'project_id is required',
+    });
+    return;
+  }
+
+  const cursor = (parseInt(query.page || '1', 10) || 1) - 1;
+  const options: GetEventListOptions = {
+    projectId,
+    events: (Array.isArray(query.event) ? query.event : [query.event]).filter(
+      (s): s is string => typeof s === 'string'
+    ),
+    startDate: query.start ? new Date(query.start) : undefined,
+    endDate: query.end ? new Date(query.end) : undefined,
+    cursor,
+    take: 50,
+    meta: false,
+    profile: true,
+  };
+
+  const [data, totalCount] = await Promise.all([
+    getEventList(options),
+    getEventsCount(options),
+  ]);
+
+  reply.send({
+    meta: {
+      // options,
+      count: data.length,
+      totalCount: totalCount,
+      pages: Math.ceil(totalCount / options.take),
+      current: cursor + 1,
+    },
+    data,
+  });
+}

apps/api/src/index.ts

@@ -1,9 +1,11 @@
 import cors from '@fastify/cors';
 import Fastify from 'fastify';
 
+import type { IServiceClient } from '@openpanel/db';
 import { redisPub } from '@openpanel/redis';
 
 import eventRouter from './routes/event.router';
+import exportRouter from './routes/export.router';
 import liveRouter from './routes/live.router';
 import miscRouter from './routes/misc.router';
 import profileRouter from './routes/profile.router';
@@ -12,6 +14,7 @@ import { logger, logInfo } from './utils/logger';
 declare module 'fastify' {
   interface FastifyRequest {
     projectId: string;
+    client: IServiceClient | null;
   }
 }
 
@@ -33,6 +36,7 @@ const startServer = async () => {
     fastify.register(profileRouter, { prefix: '/profile' });
     fastify.register(liveRouter, { prefix: '/live' });
     fastify.register(miscRouter, { prefix: '/misc' });
+    fastify.register(exportRouter, { prefix: '/export' });
     fastify.setErrorHandler((error) => {
       fastify.log.error(error);
     });

apps/api/src/routes/export.router.ts

@@ -0,0 +1,37 @@
+import * as controller from '@/controllers/export.controller';
+import { validateExportRequest } from '@/utils/auth';
+import type { FastifyPluginCallback, FastifyRequest } from 'fastify';
+
+import { Prisma } from '@openpanel/db';
+
+const eventRouter: FastifyPluginCallback = (fastify, opts, done) => {
+  fastify.addHook('preHandler', async (req: FastifyRequest, reply) => {
+    try {
+      const client = await validateExportRequest(req.headers);
+      req.client = client;
+    } catch (e) {
+      if (e instanceof Prisma.PrismaClientKnownRequestError) {
+        return reply.status(401).send({
+          error: 'Unauthorized',
+          message: 'Client ID seems to be malformed',
+        });
+      } else if (e instanceof Error) {
+        return reply
+          .status(401)
+          .send({ error: 'Unauthorized', message: e.message });
+      }
+      return reply
+        .status(401)
+        .send({ error: 'Unauthorized', message: 'Unexpected error' });
+    }
+  });
+
+  fastify.route({
+    method: 'GET',
+    url: '/events',
+    handler: controller.events,
+  });
+  done();
+};
+
+export default eventRouter;

apps/api/src/utils/auth.ts

@@ -1,7 +1,8 @@
 import type { RawRequestDefaultExpression } from 'fastify';
 
 import { verifyPassword } from '@openpanel/common';
-import { db } from '@openpanel/db';
+import type { IServiceClient } from '@openpanel/db';
+import { ClientType, db } from '@openpanel/db';
 
 import { logger } from './logger';
 
@@ -89,5 +90,39 @@ export async function validateSdkRequest(
     }
   }
 
+  if (!client.projectId) {
+    throw new Error('No project id found for client');
+  }
+
   return client.projectId;
 }
+
+export async function validateExportRequest(
+  headers: RawRequestDefaultExpression['headers']
+): Promise<IServiceClient> {
+  const clientId = headers['openpanel-client-id'] as string;
+  const clientSecret = (headers['openpanel-client-secret'] as string) || '';
+  const client = await db.client.findUnique({
+    where: {
+      id: clientId,
+    },
+  });
+
+  if (!client) {
+    throw new Error('Export: Invalid client id');
+  }
+
+  if (!client.secret) {
+    throw new Error('Export: Client has no secret');
+  }
+
+  if (client.type === ClientType.write) {
+    throw new Error('Export: Client is not allowed to export');
+  }
+
+  if (!(await verifyPassword(clientSecret, client.secret))) {
+    throw new Error('Export: Invalid client secret');
+  }
+
+  return client;
+}

apps/docs/src/pages/docs/_meta.json

@@ -19,5 +19,6 @@
   },
   "javascript": "Javascript SDK",
   "web": "Web SDK",
-  "api": "API"
+  "api": "API",
+  "export": "Export"
 }

apps/docs/src/pages/docs/export.mdx

@@ -0,0 +1,21 @@
+# Export API
+
+## Authentication
+
+To authenticate with the Export API, you need to use your `clientId` and `clientSecret`. Make sure your client has `read` or `root` mode. The default client does not have access to the Export API.
+
+We expect you to send `openpanel-client-id` and `openpanel-client-secret` headers with your requests.
+
+## Events
+
+Get all `screen_view` events from project `abc` between `2024-04-15` and `2024-04-18`.
+
+```bash
+curl 'https://api.openpanel.dev/export/events?project_id=abc&event=screen_view&start=2024-04-15&end=2024-04-18' \
+  -H 'openpanel-client-id: CLIENT_ID' \
+  -H 'openpanel-client-secret: CLIENT_SECRET'
+```
+
+## Profiles
+
+During development

packages/db/prisma/migrations/20240411180308_add_client_type/migration.sql

@@ -0,0 +1,12 @@
+-- CreateEnum
+CREATE TYPE "ClientType" AS ENUM ('read', 'write', 'root');
+
+-- DropForeignKey
+ALTER TABLE "clients" DROP CONSTRAINT "clients_projectId_fkey";
+
+-- AlterTable
+ALTER TABLE "clients" ADD COLUMN     "type" "ClientType" NOT NULL DEFAULT 'read',
+ALTER COLUMN "projectId" DROP NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "clients" ADD CONSTRAINT "clients_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "projects"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/db/prisma/migrations/20240411180951_change_default_client_type/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "clients" ALTER COLUMN "type" SET DEFAULT 'write';

packages/db/prisma/schema.prisma

@@ -90,14 +90,21 @@ model Profile {
   @@map("profiles")
 }
 
+enum ClientType {
+  read
+  write
+  root
+}
+
 model Client {
-  id               String  @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+  id               String     @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
   name             String
   secret           String?
-  projectId        String
+  type             ClientType @default(write)
-  project          Project @relation(fields: [projectId], references: [id])
+  projectId        String?
+  project          Project?   @relation(fields: [projectId], references: [id])
   organizationSlug String
-  cors             String  @default("*")
+  cors             String     @default("*")
 
   createdAt DateTime @default(now())
   updatedAt DateTime @default(now()) @updatedAt

packages/db/src/services/event.service.ts

@@ -248,6 +248,10 @@ export interface GetEventListOptions {
   cursor?: number;
   events?: string[] | null;
   filters?: IChartEventFilter[];
+  startDate?: Date;
+  endDate?: Date;
+  meta?: boolean;
+  profile?: boolean;
 }
 
 export async function getEventList({
@@ -257,6 +261,10 @@ export async function getEventList({
   profileId,
   events,
   filters,
+  startDate,
+  endDate,
+  meta = true,
+  profile = true,
 }: GetEventListOptions) {
   const { sb, getSql, join } = createSqlBuilder();
 
@@ -268,6 +276,10 @@ export async function getEventList({
     sb.where.deviceId = `device_id IN (SELECT device_id as did FROM events WHERE profile_id = ${escape(profileId)} group by did)`;
   }
 
+  if (startDate && endDate) {
+    sb.where.created_at = `created_at BETWEEN '${formatClickhouseDate(startDate)}' AND '${formatClickhouseDate(endDate)}'`;
+  }
+
   if (events && events.length > 0) {
     sb.where.events = `name IN (${join(
       events.map((event) => escape(event)),
@@ -288,7 +300,7 @@ export async function getEventList({
 
   sb.orderBy.created_at = 'created_at DESC';
 
-  return getEvents(getSql(), { profile: true, meta: true });
+  return getEvents(getSql(), { profile, meta });
 }
 
 export async function getEventsCount({
@@ -296,6 +308,8 @@ export async function getEventsCount({
   profileId,
   events,
   filters,
+  startDate,
+  endDate,
 }: Omit<GetEventListOptions, 'cursor' | 'take'>) {
   const { sb, getSql, join } = createSqlBuilder();
   sb.where.projectId = `project_id = ${escape(projectId)}`;
@@ -303,6 +317,10 @@ export async function getEventsCount({
     sb.where.profileId = `profile_id = ${escape(profileId)}`;
   }
 
+  if (startDate && endDate) {
+    sb.where.created_at = `created_at BETWEEN '${formatClickhouseDate(startDate)}' AND '${formatClickhouseDate(endDate)}'`;
+  }
+
   if (events && events.length > 0) {
     sb.where.events = `name IN (${join(
       events.map((event) => escape(event)),