feat: new importer (#214)

packages/trpc/package.json

@@ -16,6 +16,7 @@
     "@openpanel/payments": "workspace:^",
     "@openpanel/redis": "workspace:*",
     "@openpanel/validation": "workspace:*",
+    "@openpanel/queue": "workspace:*",
     "@trpc-limiter/redis": "^0.0.2",
     "@trpc/client": "^11.6.0",
     "@trpc/server": "^11.6.0",

packages/trpc/src/root.ts

@@ -4,6 +4,7 @@ import { chatRouter } from './routers/chat';
 import { clientRouter } from './routers/client';
 import { dashboardRouter } from './routers/dashboard';
 import { eventRouter } from './routers/event';
+import { importRouter } from './routers/import';
 import { integrationRouter } from './routers/integration';
 import { notificationRouter } from './routers/notification';
 import { onboardingRouter } from './routers/onboarding';
@@ -40,6 +41,7 @@ export const appRouter = createTRPCRouter({
   reference: referenceRouter,
   notification: notificationRouter,
   integration: integrationRouter,
+  import: importRouter,
   auth: authRouter,
   subscription: subscriptionRouter,
   overview: overviewRouter,

packages/trpc/src/routers/auth.ts

@@ -12,7 +12,7 @@ import {
   validateSessionToken,
   verifyPasswordHash,
 } from '@openpanel/auth';
-import { generateSecureId } from '@openpanel/common/server/id';
+import { generateSecureId } from '@openpanel/common/server';
 import {
   connectUserToOrganization,
   db,

packages/trpc/src/routers/import.ts

@@ -0,0 +1,178 @@
+import { z } from 'zod';
+
+import { db } from '@openpanel/db';
+import { importQueue } from '@openpanel/queue';
+import { zCreateImport } from '@openpanel/validation';
+
+import { getProjectAccess } from '../access';
+import { TRPCAccessError } from '../errors';
+import { createTRPCRouter, protectedProcedure } from '../trpc';
+
+export const importRouter = createTRPCRouter({
+  list: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      const access = await getProjectAccess({
+        projectId: input.projectId,
+        userId: ctx.session.userId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this project');
+      }
+
+      return db.import.findMany({
+        where: {
+          projectId: input.projectId,
+        },
+        orderBy: {
+          createdAt: 'desc',
+        },
+      });
+    }),
+
+  get: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .query(async ({ input, ctx }) => {
+      const importRecord = await db.import.findUniqueOrThrow({
+        where: {
+          id: input.id,
+        },
+        include: {
+          project: true,
+        },
+      });
+
+      const access = await getProjectAccess({
+        projectId: importRecord.projectId,
+        userId: ctx.session.userId,
+      });
+
+      if (!access) {
+        throw TRPCAccessError('You do not have access to this import');
+      }
+
+      return importRecord;
+    }),
+
+  create: protectedProcedure
+    .input(zCreateImport)
+    .mutation(async ({ input, ctx }) => {
+      const access = await getProjectAccess({
+        projectId: input.projectId,
+        userId: ctx.session.userId,
+      });
+
+      if (!access || (typeof access !== 'boolean' && access.level === 'read')) {
+        throw TRPCAccessError(
+          'You do not have permission to create imports for this project',
+        );
+      }
+
+      // Create import record
+      const importRecord = await db.import.create({
+        data: {
+          projectId: input.projectId,
+          config: input.config,
+          status: 'pending',
+        },
+      });
+
+      // Add job to queue
+      const job = await importQueue.add('import', {
+        type: 'import',
+        payload: {
+          importId: importRecord.id,
+        },
+      });
+
+      // Update import record with job ID
+      await db.import.update({
+        where: { id: importRecord.id },
+        data: { jobId: job.id },
+      });
+
+      return {
+        ...importRecord,
+        jobId: job.id,
+      };
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const importRecord = await db.import.findUniqueOrThrow({
+        where: {
+          id: input.id,
+        },
+      });
+
+      const access = await getProjectAccess({
+        projectId: importRecord.projectId,
+        userId: ctx.session.userId,
+      });
+
+      if (!access || (typeof access !== 'boolean' && access.level === 'read')) {
+        throw TRPCAccessError(
+          'You do not have permission to delete imports for this project',
+        );
+      }
+
+      if (importRecord.jobId) {
+        const job = await importQueue.getJob(importRecord.jobId);
+        if (job) {
+          await job.remove();
+        }
+      }
+
+      return db.import.delete({
+        where: {
+          id: input.id,
+        },
+      });
+    }),
+
+  retry: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const importRecord = await db.import.findUniqueOrThrow({
+        where: {
+          id: input.id,
+        },
+      });
+
+      const access = await getProjectAccess({
+        projectId: importRecord.projectId,
+        userId: ctx.session.userId,
+      });
+
+      if (!access || (typeof access !== 'boolean' && access.level === 'read')) {
+        throw TRPCAccessError(
+          'You do not have permission to retry imports for this project',
+        );
+      }
+
+      // Only allow retry for failed imports
+      if (importRecord.status !== 'failed') {
+        throw new Error('Only failed imports can be retried');
+      }
+
+      // Add new job to queue
+      const job = await importQueue.add('import', {
+        type: 'import',
+        payload: {
+          importId: importRecord.id,
+        },
+      });
+
+      // Update import record
+      return db.import.update({
+        where: { id: importRecord.id },
+        data: {
+          jobId: job.id,
+          status: 'pending',
+          errorMessage: null,
+        },
+      });
+    }),
+});

packages/trpc/src/routers/organization.ts

@@ -11,7 +11,7 @@ import {
 } from '@openpanel/db';
 import { zEditOrganization, zInviteUser } from '@openpanel/validation';
 
-import { generateSecureId } from '@openpanel/common/server/id';
+import { generateSecureId } from '@openpanel/common/server';
 import { sendEmail } from '@openpanel/email';
 import { addDays } from 'date-fns';
 import { getOrganizationAccess } from '../access';