feat: new importer (#214)

2025-11-05 09:49:36 +01:00
parent b51bc8f3f6
commit 212254d31a
80 changed files with 4884 additions and 842 deletions
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -64,7 +64,7 @@
    "@types/uuid": "^10.0.0",
    "@types/ws": "^8.5.14",
    "js-yaml": "^4.1.0",
-    "tsdown": "^0.14.2",
+    "tsdown": "0.14.2",
    "typescript": "catalog:"
  }
 }
--- a/apps/start/Dockerfile
+++ b/apps/start/Dockerfile
@@ -29,6 +29,7 @@ COPY apps/start/package.json ./apps/start/
 COPY packages/trpc/package.json packages/trpc/
 COPY packages/json/package.json packages/json/
 COPY packages/common/package.json packages/common/
+COPY packages/importer/package.json packages/importer/
 COPY packages/payments/package.json packages/payments/
 COPY packages/constants/package.json packages/constants/
 COPY packages/validation/package.json packages/validation/
@@ -85,6 +86,7 @@ COPY --from=build /app/packages/trpc/package.json ./packages/trpc/
 COPY --from=build /app/packages/auth/package.json ./packages/auth/
 COPY --from=build /app/packages/json/package.json ./packages/json/
 COPY --from=build /app/packages/common/package.json ./packages/common/
+COPY --from=build /app/packages/importer/package.json ./packages/importer/
 COPY --from=build /app/packages/payments/package.json ./packages/payments/
 COPY --from=build /app/packages/constants/package.json ./packages/constants/
 COPY --from=build /app/packages/validation/package.json ./packages/validation/
--- a/apps/start/package.json
+++ b/apps/start/package.json
@@ -8,7 +8,7 @@
    "preview": "vite preview",
    "deploy": "npx wrangler deploy",
    "cf-typegen": "wrangler types",
-    "build": "vite build",
+    "build": "pnpm with-env vite build",
    "serve": "vite preview",
    "test": "vitest run",
    "format": "biome format",
@@ -30,6 +30,7 @@
    "@openpanel/common": "workspace:^",
    "@openpanel/constants": "workspace:^",
    "@openpanel/integrations": "workspace:^",
+    "@openpanel/importer": "workspace:^",
    "@openpanel/json": "workspace:*",
    "@openpanel/payments": "workspace:*",
    "@openpanel/sdk-info": "workspace:^",
--- a/apps/start/src/components/integrations/integration-card.tsx
+++ b/apps/start/src/components/integrations/integration-card.tsx
@@ -50,12 +50,15 @@ export function IntegrationCardHeaderButtons({
 export function IntegrationCardLogoImage({
  src,
  backgroundColor,
+  className,
 }: {
  src: string;
  backgroundColor: string;
+  className?: string;
 }) {
  return (
    <IntegrationCardLogo
+      className={className}
      style={{
        backgroundColor,
      }}
--- a/apps/start/src/components/project-mapper.tsx
+++ b/apps/start/src/components/project-mapper.tsx
@@ -0,0 +1,116 @@
+import { Button } from '@/components/ui/button';
+import { Input } from '@/components/ui/input';
+import { Label } from '@/components/ui/label';
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from '@/components/ui/select';
+import type { IServiceProject } from '@openpanel/db';
+import { PlusIcon, XIcon } from 'lucide-react';
+import type {
+  Control,
+  FieldArrayWithId,
+  UseFieldArrayAppend,
+  UseFieldArrayRemove,
+  UseFormRegister,
+  UseFormSetValue,
+  UseFormWatch,
+} from 'react-hook-form';
+
+interface ProjectMapperProps {
+  fields: FieldArrayWithId<any, 'projectMapper', 'id'>[];
+  append: UseFieldArrayAppend<any, 'projectMapper'>;
+  remove: UseFieldArrayRemove;
+  projects: IServiceProject[];
+  register: UseFormRegister<any>;
+  watch: UseFormWatch<any>;
+  setValue: UseFormSetValue<any>;
+}
+
+export function ProjectMapper({
+  fields,
+  append,
+  remove,
+  projects,
+  register,
+  watch,
+  setValue,
+}: ProjectMapperProps) {
+  return (
+    <div className="space-y-2">
+      <div className="flex items-center justify-between">
+        <Label className="mb-0">Project Mapper (Optional)</Label>
+        <Button
+          type="button"
+          variant="outline"
+          size="sm"
+          onClick={() => append({ from: '', to: '' })}
+        >
+          <PlusIcon className="mr-1 h-4 w-4" />
+          Add Mapping
+        </Button>
+      </div>
+      {fields.length === 0 && (
+        <p className="text-sm text-muted-foreground leading-normal">
+          Map source project IDs to your OpenPanel projects. If you skip mapping
+          all data will be imported to your current project.
+        </p>
+      )}
+
+      {fields.length > 0 && (
+        <div className="space-y-2">
+          {fields.map((field, index) => (
+            <div key={field.id} className="gap-2 rounded-md border p-3 row">
+              <div className="grid grid-cols-2 gap-2">
+                <div className="flex-1">
+                  <Label className="text-xs text-muted-foreground">
+                    From (Source Project ID)
+                  </Label>
+                  <Input
+                    placeholder="e.g., abc123"
+                    {...register(`projectMapper.${index}.from`)}
+                    className="mt-1"
+                  />
+                </div>
+                <div className="flex-1">
+                  <Label className="text-xs text-muted-foreground">
+                    To (OpenPanel Project)
+                  </Label>
+                  <Select
+                    value={watch(`projectMapper.${index}.to`)}
+                    onValueChange={(value) =>
+                      setValue(`projectMapper.${index}.to`, value)
+                    }
+                  >
+                    <SelectTrigger className="mt-1 w-full" size="sm">
+                      <SelectValue placeholder="Select project" />
+                    </SelectTrigger>
+                    <SelectContent>
+                      {projects.slice(0, 10).map((project) => (
+                        <SelectItem key={project.id} value={project.id}>
+                          {project.name}
+                        </SelectItem>
+                      ))}
+                    </SelectContent>
+                  </Select>
+                </div>
+              </div>
+              <Button
+                type="button"
+                variant="ghost"
+                size="icon"
+                onClick={() => remove(index)}
+                className="mt-5"
+              >
+                <XIcon className="h-4 w-4" />
+              </Button>
+            </div>
+          ))}
+        </div>
+      )}
+    </div>
+  );
+}
--- a/apps/start/src/components/ui/badge.tsx
+++ b/apps/start/src/components/ui/badge.tsx
@@ -9,6 +9,11 @@ const badgeVariants = cva(
  {
    variants: {
      variant: {
+        success:
+          'border-transparent bg-emerald-700 text-white [a&]:hover:bg-emerald-700/90',
+        warning:
+          'border-transparent bg-yellow-500 text-white [a&]:hover:bg-yellow-500/90',
+        info: 'border-transparent bg-blue-500 text-white [a&]:hover:bg-blue-500/90',
        default:
          'border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90',
        secondary:
--- a/apps/start/src/modals/add-import.tsx
+++ b/apps/start/src/modals/add-import.tsx
@@ -0,0 +1,282 @@
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { toast } from 'sonner';
+
+import { InputWithLabel, WithLabel } from '@/components/forms/input-with-label';
+import { ProjectMapper } from '@/components/project-mapper';
+import { Button } from '@/components/ui/button';
+import { useAppParams } from '@/hooks/use-app-params';
+import { useTRPC } from '@/integrations/trpc/react';
+import { cn } from '@/lib/utils';
+import { zodResolver } from '@hookform/resolvers/zod';
+import type {
+  IImportConfig,
+  IMixpanelImportConfig,
+  IUmamiImportConfig,
+} from '@openpanel/validation';
+import {
+  zMixpanelImportConfig,
+  zUmamiImportConfig,
+} from '@openpanel/validation';
+import { format } from 'date-fns';
+import { CalendarIcon } from 'lucide-react';
+import { useFieldArray, useForm } from 'react-hook-form';
+import type { z } from 'zod';
+import { popModal, pushModal } from '.';
+import { ModalContent, ModalHeader } from './Modal/Container';
+
+type Provider = 'umami' | 'plausible' | 'mixpanel';
+
+interface AddImportProps {
+  provider: Provider;
+  name: string;
+  types: ('file' | 'api')[];
+}
+
+type UmamiFormData = z.infer<typeof zUmamiImportConfig>;
+type MixpanelFormData = z.infer<typeof zMixpanelImportConfig>;
+
+interface UmamiImportProps {
+  onSubmit: (config: IUmamiImportConfig) => void;
+  isPending: boolean;
+  organizationId: string;
+}
+
+function UmamiImport({
+  onSubmit,
+  isPending,
+  organizationId,
+}: UmamiImportProps) {
+  const trpc = useTRPC();
+  const { data: projects = [] } = useQuery(
+    trpc.project.list.queryOptions({
+      organizationId,
+    }),
+  );
+
+  const form = useForm<UmamiFormData>({
+    resolver: zodResolver(zUmamiImportConfig),
+    defaultValues: {
+      provider: 'umami',
+      type: 'file',
+      fileUrl: '',
+      projectMapper: [],
+    },
+  });
+
+  const { fields, append, remove } = useFieldArray({
+    control: form.control,
+    name: 'projectMapper',
+  });
+
+  const handleSubmit = form.handleSubmit((data) => {
+    onSubmit(data);
+  });
+
+  return (
+    <form onSubmit={handleSubmit} className="space-y-4">
+      <div className="space-y-4 py-4">
+        <InputWithLabel
+          label="File URL"
+          placeholder="https://example.com/export.csv"
+          error={form.formState.errors.fileUrl?.message}
+          info="Provide a publicly accessible URL to your exported CSV file."
+          {...form.register('fileUrl')}
+        />
+
+        <ProjectMapper
+          fields={fields}
+          append={append}
+          remove={remove}
+          projects={projects}
+          register={form.register}
+          watch={form.watch}
+          setValue={form.setValue}
+        />
+      </div>
+
+      <div className="flex justify-between">
+        <Button type="button" variant="outline" onClick={() => popModal()}>
+          Cancel
+        </Button>
+        <Button type="submit" disabled={isPending}>
+          {isPending ? 'Starting...' : 'Start Import'}
+        </Button>
+      </div>
+    </form>
+  );
+}
+
+interface MixpanelImportProps {
+  onSubmit: (config: IMixpanelImportConfig) => void;
+  isPending: boolean;
+  organizationId: string;
+}
+
+function MixpanelImport({
+  onSubmit,
+  isPending,
+  organizationId,
+}: MixpanelImportProps) {
+  const trpc = useTRPC();
+  const form = useForm<MixpanelFormData>({
+    resolver: zodResolver(zMixpanelImportConfig),
+    defaultValues: {
+      provider: 'mixpanel',
+      type: 'api',
+      serviceAccount: '',
+      serviceSecret: '',
+      projectId: '',
+      from: '',
+      to: '',
+    },
+  });
+
+  const handleDateRangeSelect = () => {
+    pushModal('DateRangerPicker', {
+      startDate: form.getValues('from')
+        ? new Date(form.getValues('from'))
+        : undefined,
+      endDate: form.getValues('to')
+        ? new Date(form.getValues('to'))
+        : undefined,
+      onChange: ({ startDate, endDate }) => {
+        form.setValue('from', format(startDate, 'yyyy-MM-dd'));
+        form.setValue('to', format(endDate, 'yyyy-MM-dd'));
+        form.trigger('from');
+        form.trigger('to');
+      },
+    });
+  };
+
+  const handleSubmit = form.handleSubmit((data) => {
+    onSubmit(data);
+  });
+
+  return (
+    <form onSubmit={handleSubmit} className="space-y-4">
+      <div className="space-y-4 py-4">
+        <InputWithLabel
+          label="Service Account"
+          placeholder="Eg. xxx.xxx.mp-service-account"
+          error={form.formState.errors.serviceAccount?.message}
+          {...form.register('serviceAccount')}
+        />
+
+        <InputWithLabel
+          label="Service Secret"
+          type="password"
+          placeholder="Your Mixpanel service secret"
+          error={form.formState.errors.serviceSecret?.message}
+          {...form.register('serviceSecret')}
+        />
+
+        <InputWithLabel
+          label="Project ID"
+          placeholder="Your Mixpanel project ID"
+          error={form.formState.errors.projectId?.message}
+          {...form.register('projectId')}
+        />
+
+        <WithLabel
+          label="Date Range"
+          info={
+            !form.getValues('from') || !form.getValues('to')
+              ? 'Select the date range for importing data'
+              : undefined
+          }
+        >
+          <Button
+            type="button"
+            variant="outline"
+            className={cn(
+              'w-full justify-start text-left font-normal',
+              (!form.getValues('from') || !form.getValues('to')) &&
+                'text-muted-foreground',
+            )}
+            onClick={handleDateRangeSelect}
+          >
+            <CalendarIcon className="mr-2 h-4 w-4" />
+            {form.getValues('from') && form.getValues('to') ? (
+              <>
+                {format(new Date(form.getValues('from')), 'LLL dd, y')} -{' '}
+                {format(new Date(form.getValues('to')), 'LLL dd, y')}
+              </>
+            ) : (
+              <span>Pick a date range</span>
+            )}
+          </Button>
+        </WithLabel>
+
+        <InputWithLabel
+          label="Screen View Property"
+          placeholder="Enter the name of the property that contains the screen name"
+          info="Leave empty if not applicable"
+          error={form.formState.errors.mapScreenViewProperty?.message}
+          {...form.register('mapScreenViewProperty')}
+        />
+      </div>
+
+      <div className="flex justify-between">
+        <Button type="button" variant="outline" onClick={() => popModal()}>
+          Cancel
+        </Button>
+        <Button type="submit" disabled={isPending}>
+          {isPending ? 'Starting...' : 'Start Import'}
+        </Button>
+      </div>
+    </form>
+  );
+}
+
+export default function AddImport({ provider, name }: AddImportProps) {
+  const { projectId, organizationId } = useAppParams();
+  const trpc = useTRPC();
+  const queryClient = useQueryClient();
+
+  const createImport = useMutation(
+    trpc.import.create.mutationOptions({
+      onSuccess() {
+        toast.success('Import started', {
+          description: 'Your data import has been queued for processing.',
+        });
+        popModal();
+        queryClient.invalidateQueries(trpc.import.list.pathFilter());
+      },
+      onError: (error) => {
+        toast.error('Import failed', {
+          description: error.message,
+        });
+      },
+    }),
+  );
+
+  const handleImportSubmit = (config: IImportConfig) => {
+    createImport.mutate({
+      projectId,
+      provider: config.provider,
+      config,
+    });
+  };
+
+  return (
+    <ModalContent>
+      <ModalHeader title={`Import from ${name}`} />
+
+      {provider === 'umami' && (
+        <UmamiImport
+          onSubmit={handleImportSubmit}
+          isPending={createImport.isPending}
+          organizationId={organizationId}
+        />
+      )}
+
+      {provider === 'mixpanel' && (
+        <MixpanelImport
+          onSubmit={handleImportSubmit}
+          isPending={createImport.isPending}
+          organizationId={organizationId}
+        />
+      )}
+    </ModalContent>
+  );
+}
--- a/apps/start/src/modals/date-ranger-picker.tsx
+++ b/apps/start/src/modals/date-ranger-picker.tsx
@@ -27,6 +27,7 @@ export default function DateRangerPicker({
  return (
    <ModalContent className="p-4 md:p-8 min-w-fit">
      <Calendar
+        captionLayout="dropdown"
        initialFocus
        mode="range"
        defaultMonth={subMonths(
--- a/apps/start/src/modals/index.tsx
+++ b/apps/start/src/modals/index.tsx
@@ -5,6 +5,7 @@ import OverviewTopPagesModal from '@/components/overview/overview-top-pages-moda
 import Instructions from './Instructions';
 import AddClient from './add-client';
 import AddDashboard from './add-dashboard';
+import AddImport from './add-import';
 import AddIntegration from './add-integration';
 import AddNotificationRule from './add-notification-rule';
 import AddProject from './add-project';
@@ -38,6 +39,7 @@ const modals = {
  EditClient: EditClient,
  AddProject: AddProject,
  AddClient: AddClient,
+  AddImport: AddImport,
  Confirm: Confirm,
  SaveReport: SaveReport,
  AddDashboard: AddDashboard,
--- a/apps/start/src/routeTree.gen.ts
+++ b/apps/start/src/routeTree.gen.ts
@@ -56,6 +56,7 @@ import { Route as AppOrganizationIdProjectIdSettingsTabsIndexRouteImport } from
 import { Route as AppOrganizationIdProjectIdProfilesTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.profiles._tabs.index'
 import { Route as AppOrganizationIdProjectIdNotificationsTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.notifications._tabs.index'
 import { Route as AppOrganizationIdProjectIdEventsTabsIndexRouteImport } from './routes/_app.$organizationId.$projectId_.events._tabs.index'
+import { Route as AppOrganizationIdProjectIdSettingsTabsImportsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.imports'
 import { Route as AppOrganizationIdProjectIdSettingsTabsEventsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.events'
 import { Route as AppOrganizationIdProjectIdSettingsTabsDetailsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.details'
 import { Route as AppOrganizationIdProjectIdSettingsTabsClientsRouteImport } from './routes/_app.$organizationId.$projectId_.settings._tabs.clients'
@@ -381,6 +382,12 @@ const AppOrganizationIdProjectIdEventsTabsIndexRoute =
    path: '/',
    getParentRoute: () => AppOrganizationIdProjectIdEventsTabsRoute,
  } as any)
+const AppOrganizationIdProjectIdSettingsTabsImportsRoute =
+  AppOrganizationIdProjectIdSettingsTabsImportsRouteImport.update({
+    id: '/imports',
+    path: '/imports',
+    getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute,
+  } as any)
 const AppOrganizationIdProjectIdSettingsTabsEventsRoute =
  AppOrganizationIdProjectIdSettingsTabsEventsRouteImport.update({
    id: '/events',
@@ -515,6 +522,7 @@ export interface FileRoutesByFullPath {
  '/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
  '/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
  '/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+  '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
  '/$organizationId/$projectId/events/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute
  '/$organizationId/$projectId/notifications/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute
  '/$organizationId/$projectId/profiles/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute
@@ -569,6 +577,7 @@ export interface FileRoutesByTo {
  '/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
  '/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
  '/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+  '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
  '/$organizationId/$projectId/profiles/$profileId/events': typeof AppOrganizationIdProjectIdProfilesProfileIdTabsEventsRoute
 }
 export interface FileRoutesById {
@@ -633,6 +642,7 @@ export interface FileRoutesById {
  '/_app/$organizationId/$projectId_/settings/_tabs/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
  '/_app/$organizationId/$projectId_/settings/_tabs/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
  '/_app/$organizationId/$projectId_/settings/_tabs/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+  '/_app/$organizationId/$projectId_/settings/_tabs/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
  '/_app/$organizationId/$projectId_/events/_tabs/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute
  '/_app/$organizationId/$projectId_/notifications/_tabs/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute
  '/_app/$organizationId/$projectId_/profiles/_tabs/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute
@@ -692,6 +702,7 @@ export interface FileRouteTypes {
    | '/$organizationId/$projectId/settings/clients'
    | '/$organizationId/$projectId/settings/details'
    | '/$organizationId/$projectId/settings/events'
+    | '/$organizationId/$projectId/settings/imports'
    | '/$organizationId/$projectId/events/'
    | '/$organizationId/$projectId/notifications/'
    | '/$organizationId/$projectId/profiles/'
@@ -746,6 +757,7 @@ export interface FileRouteTypes {
    | '/$organizationId/$projectId/settings/clients'
    | '/$organizationId/$projectId/settings/details'
    | '/$organizationId/$projectId/settings/events'
+    | '/$organizationId/$projectId/settings/imports'
    | '/$organizationId/$projectId/profiles/$profileId/events'
  id:
    | '__root__'
@@ -809,6 +821,7 @@ export interface FileRouteTypes {
    | '/_app/$organizationId/$projectId_/settings/_tabs/clients'
    | '/_app/$organizationId/$projectId_/settings/_tabs/details'
    | '/_app/$organizationId/$projectId_/settings/_tabs/events'
+    | '/_app/$organizationId/$projectId_/settings/_tabs/imports'
    | '/_app/$organizationId/$projectId_/events/_tabs/'
    | '/_app/$organizationId/$projectId_/notifications/_tabs/'
    | '/_app/$organizationId/$projectId_/profiles/_tabs/'
@@ -1194,6 +1207,13 @@ declare module '@tanstack/react-router' {
      preLoaderRoute: typeof AppOrganizationIdProjectIdEventsTabsIndexRouteImport
      parentRoute: typeof AppOrganizationIdProjectIdEventsTabsRoute
    }
+    '/_app/$organizationId/$projectId_/settings/_tabs/imports': {
+      id: '/_app/$organizationId/$projectId_/settings/_tabs/imports'
+      path: '/imports'
+      fullPath: '/$organizationId/$projectId/settings/imports'
+      preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRouteImport
+      parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute
+    }
    '/_app/$organizationId/$projectId_/settings/_tabs/events': {
      id: '/_app/$organizationId/$projectId_/settings/_tabs/events'
      path: '/events'
@@ -1521,6 +1541,7 @@ interface AppOrganizationIdProjectIdSettingsTabsRouteChildren {
  AppOrganizationIdProjectIdSettingsTabsClientsRoute: typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute
  AppOrganizationIdProjectIdSettingsTabsDetailsRoute: typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute
  AppOrganizationIdProjectIdSettingsTabsEventsRoute: typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute
+  AppOrganizationIdProjectIdSettingsTabsImportsRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute
  AppOrganizationIdProjectIdSettingsTabsIndexRoute: typeof AppOrganizationIdProjectIdSettingsTabsIndexRoute
 }

@@ -1532,6 +1553,8 @@ const AppOrganizationIdProjectIdSettingsTabsRouteChildren: AppOrganizationIdProj
      AppOrganizationIdProjectIdSettingsTabsDetailsRoute,
    AppOrganizationIdProjectIdSettingsTabsEventsRoute:
      AppOrganizationIdProjectIdSettingsTabsEventsRoute,
+    AppOrganizationIdProjectIdSettingsTabsImportsRoute:
+      AppOrganizationIdProjectIdSettingsTabsImportsRoute,
    AppOrganizationIdProjectIdSettingsTabsIndexRoute:
      AppOrganizationIdProjectIdSettingsTabsIndexRoute,
  }
--- a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx
@@ -0,0 +1,303 @@
+import { FullPageEmptyState } from '@/components/full-page-empty-state';
+import {
+  IntegrationCard,
+  IntegrationCardFooter,
+  IntegrationCardLogoImage,
+} from '@/components/integrations/integration-card';
+import { Skeleton } from '@/components/skeleton';
+import { Badge } from '@/components/ui/badge';
+import { Button } from '@/components/ui/button';
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableHeader,
+  TableRow,
+} from '@/components/ui/table';
+import { Tooltiper } from '@/components/ui/tooltip';
+import { useAppParams } from '@/hooks/use-app-params';
+import { useTRPC } from '@/integrations/trpc/react';
+import { pushModal } from '@/modals';
+import { IMPORT_PROVIDERS } from '@openpanel/importer/providers';
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
+import { createFileRoute } from '@tanstack/react-router';
+import { formatDistanceToNow } from 'date-fns';
+import {
+  CheckCircleIcon,
+  Download,
+  InfoIcon,
+  Loader2Icon,
+  XCircleIcon,
+} from 'lucide-react';
+import { toast } from 'sonner';
+
+export const Route = createFileRoute(
+  '/_app/$organizationId/$projectId_/settings/_tabs/imports',
+)({
+  component: ImportsSettings,
+});
+
+function ImportsSettings() {
+  const { projectId } = useAppParams();
+  const trpc = useTRPC();
+  const queryClient = useQueryClient();
+
+  const importsQuery = useQuery(
+    trpc.import.list.queryOptions(
+      { projectId },
+      {
+        refetchInterval: 5000,
+      },
+    ),
+  );
+  const imports = importsQuery.data ?? [];
+
+  const deleteImport = useMutation(
+    trpc.import.delete.mutationOptions({
+      onSuccess: () => {
+        toast.success('Import deleted', {
+          description: 'The import has been successfully deleted.',
+        });
+        queryClient.invalidateQueries(trpc.import.list.pathFilter());
+      },
+    }),
+  );
+
+  const retryImport = useMutation(
+    trpc.import.retry.mutationOptions({
+      onSuccess: () => {
+        toast.success('Import retried', {
+          description: 'The import has been queued for processing again.',
+        });
+        queryClient.invalidateQueries(trpc.import.list.pathFilter());
+      },
+    }),
+  );
+
+  const handleProviderSelect = (
+    provider: (typeof IMPORT_PROVIDERS)[number],
+  ) => {
+    pushModal('AddImport', {
+      provider: provider.id,
+      name: provider.name,
+      types: provider.types,
+    });
+  };
+
+  const getStatusBadge = (status: string, errorMessage: string | null) => {
+    const variants: Record<string, any> = {
+      pending: 'secondary',
+      processing: 'default',
+      completed: 'success',
+      failed: 'destructive',
+    };
+    const icons: Record<string, React.ReactNode> = {
+      pending: <Loader2Icon className="w-4 h-4 animate-spin" />,
+      processing: <Loader2Icon className="w-4 h-4 animate-spin" />,
+      completed: <CheckCircleIcon className="w-4 h-4" />,
+      failed: <XCircleIcon className="w-4 h-4" />,
+    };
+
+    if (status === 'failed') {
+      return (
+        <Tooltiper
+          content={errorMessage}
+          tooltipClassName="max-w-xs break-words"
+        >
+          <Badge variant={variants[status] || 'default'} className="capitalize">
+            {icons[status] || null}
+            {status}
+          </Badge>
+        </Tooltiper>
+      );
+    }
+
+    return (
+      <Badge variant={variants[status] || 'default'} className="capitalize">
+        {icons[status] || null}
+        {status}
+      </Badge>
+    );
+  };
+
+  return (
+    <div className="space-y-8">
+      <div>
+        <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
+          {IMPORT_PROVIDERS.map((provider) => (
+            <IntegrationCard
+              key={provider.id}
+              icon={
+                <IntegrationCardLogoImage
+                  src={provider.logo}
+                  backgroundColor={provider.backgroundColor}
+                  className="p-4"
+                />
+              }
+              name={provider.name}
+              description={provider.description}
+            >
+              <IntegrationCardFooter className="row justify-end">
+                <Button
+                  variant="ghost"
+                  onClick={() => handleProviderSelect(provider)}
+                >
+                  <Download className="w-4 h-4 mr-2" />
+                  Import Data
+                </Button>
+              </IntegrationCardFooter>
+            </IntegrationCard>
+          ))}
+        </div>
+      </div>
+
+      <div>
+        <h3 className="text-lg font-medium mb-4">Import History</h3>
+
+        <div className="border rounded-lg">
+          <Table>
+            <TableHeader>
+              <TableRow>
+                <TableHead>Provider</TableHead>
+                <TableHead>Created</TableHead>
+                <TableHead>Status</TableHead>
+                <TableHead>Progress</TableHead>
+                <TableHead>Config</TableHead>
+                <TableHead className="text-right">Actions</TableHead>
+              </TableRow>
+            </TableHeader>
+            <TableBody>
+              {!importsQuery.isLoading && imports.length === 0 && (
+                <TableRow>
+                  <TableCell colSpan={6}>
+                    <FullPageEmptyState
+                      title="No imports yet"
+                      description="Your import history will appear here."
+                    />
+                  </TableCell>
+                </TableRow>
+              )}
+              {importsQuery.isLoading &&
+                [1, 2, 3, 4].map((index) => (
+                  <TableRow key={index}>
+                    <TableCell>
+                      <Skeleton className="h-4 w-3/5" />
+                    </TableCell>
+                    <TableCell>
+                      <Skeleton className="h-4 w-3/5" />
+                    </TableCell>
+                    <TableCell>
+                      <Skeleton className="h-4 w-3/5" />
+                    </TableCell>
+                    <TableCell>
+                      <Skeleton className="h-4 w-3/5" />
+                    </TableCell>
+                    <TableCell>
+                      <Skeleton className="h-4 w-3/5" />
+                    </TableCell>
+                    <TableCell className="text-right justify-end row">
+                      <Skeleton className="h-4 w-3/5" />
+                    </TableCell>
+                  </TableRow>
+                ))}
+              {imports.map((imp) => (
+                <TableRow key={imp.id}>
+                  <TableCell className="font-medium capitalize">
+                    <div className="row gap-2 items-center">
+                      <div>{imp.config.provider}</div>
+                      <Badge variant="outline" className="uppercase">
+                        {imp.config.type}
+                      </Badge>
+                    </div>
+                  </TableCell>
+                  <TableCell>
+                    {formatDistanceToNow(new Date(imp.createdAt), {
+                      addSuffix: true,
+                    })}
+                  </TableCell>
+                  <TableCell>
+                    <div className="space-y-1">
+                      {getStatusBadge(imp.status, imp.errorMessage)}
+                      {imp.statusMessage && (
+                        <div className="text-xs text-muted-foreground truncate">
+                          {imp.statusMessage}
+                        </div>
+                      )}
+                    </div>
+                  </TableCell>
+                  <TableCell>
+                    {imp.totalEvents > 0 ? (
+                      <div className="space-y-1 font-mono">
+                        <div className="text-sm">
+                          {imp.processedEvents.toLocaleString()}
+                          {' / '}
+                          <Tooltiper
+                            content="Estimated number of events. Can be inaccurate depending on the provider."
+                            tooltipClassName="max-w-xs"
+                          >
+                            {imp.totalEvents.toLocaleString()}{' '}
+                            <InfoIcon className="w-4 h-4 inline-block relative -top-px" />
+                          </Tooltiper>
+                        </div>
+                        {imp.status === 'processing' && (
+                          <div className="w-full bg-secondary rounded-full h-1.5">
+                            <div
+                              className="bg-primary h-1.5 rounded-full transition-all"
+                              style={{
+                                width: `${Math.min(Math.round((imp.processedEvents / imp.totalEvents) * 100), 100)}%`,
+                              }}
+                            />
+                          </div>
+                        )}
+                      </div>
+                    ) : imp.totalEvents === -1 ? (
+                      <div className="font-mono text-sm">
+                        {imp.processedEvents.toLocaleString()}
+                        {' / '}
+                        N/A
+                      </div>
+                    ) : (
+                      '-'
+                    )}
+                  </TableCell>
+
+                  <TableCell>
+                    <Tooltiper
+                      content={
+                        <pre className="font-mono text-sm leading-normal whitespace-pre-wrap break-words">
+                          {JSON.stringify(imp.config, null, 2)}
+                        </pre>
+                      }
+                      tooltipClassName="max-w-xs"
+                    >
+                      <Badge>Config</Badge>
+                    </Tooltiper>
+                  </TableCell>
+                  <TableCell className="text-right space-x-2">
+                    {imp.status === 'failed' && (
+                      <Button
+                        variant="outline"
+                        size="sm"
+                        onClick={() => retryImport.mutate({ id: imp.id })}
+                      >
+                        Retry
+                      </Button>
+                    )}
+                    <Button
+                      variant="ghost"
+                      size="sm"
+                      onClick={() => deleteImport.mutate({ id: imp.id })}
+                    >
+                      Delete
+                    </Button>
+                  </TableCell>
+                </TableRow>
+              ))}
+            </TableBody>
+          </Table>
+        </div>
+      </div>
+    </div>
+  );
+}
--- a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx
+++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx
@@ -42,6 +42,7 @@ function ProjectDashboard() {
    { id: 'details', label: 'Details' },
    { id: 'events', label: 'Events' },
    { id: 'clients', label: 'Clients' },
+    { id: 'imports', label: 'Imports' },
  ];

  const handleTabChange = (tabId: string) => {
--- a/apps/start/src/utils/math.ts
+++ b/apps/start/src/utils/math.ts
@@ -1 +1 @@
-export * from '@openpanel/common/src/math';
+export * from '@openpanel/common';
--- a/apps/start/src/utils/op.ts
+++ b/apps/start/src/utils/op.ts
@@ -1,8 +1,20 @@
 import { OpenPanel } from '@openpanel/web';

-export const op = new OpenPanel({
-  clientId: import.meta.env.VITE_OP_CLIENT_ID,
-  trackScreenViews: true,
-  trackOutgoingLinks: true,
-  trackAttributes: true,
-});
+const clientId = import.meta.env.VITE_OP_CLIENT_ID;
+
+const createOpInstance = () => {
+  if (!clientId || clientId === 'undefined') {
+    return new Proxy({} as OpenPanel, {
+      get: () => () => {},
+    });
+  }
+
+  return new OpenPanel({
+    clientId,
+    trackScreenViews: true,
+    trackOutgoingLinks: true,
+    trackAttributes: true,
+  });
+};
+
+export const op = createOpInstance();
--- a/apps/start/src/utils/slug.ts
+++ b/apps/start/src/utils/slug.ts
@@ -1 +1 @@
-export * from '@openpanel/common/src/slug';
+export * from '@openpanel/common';
--- a/apps/worker/Dockerfile
+++ b/apps/worker/Dockerfile
@@ -35,6 +35,7 @@ COPY packages/redis/package.json ./packages/redis/
 COPY packages/queue/package.json ./packages/queue/
 COPY packages/logger/package.json ./packages/logger/
 COPY packages/common/package.json ./packages/common/
+COPY packages/importer/package.json ./packages/importer/
 COPY packages/constants/package.json ./packages/constants/
 COPY packages/validation/package.json ./packages/validation/
 COPY packages/integrations/package.json packages/integrations/
@@ -80,9 +81,10 @@ COPY --from=build /app/packages/geo ./packages/geo
 COPY --from=build /app/packages/json ./packages/json
 COPY --from=build /app/packages/email ./packages/email
 COPY --from=build /app/packages/redis ./packages/redis
-COPY --from=build /app/packages/logger ./packages/logger
 COPY --from=build /app/packages/queue ./packages/queue
+COPY --from=build /app/packages/logger ./packages/logger
 COPY --from=build /app/packages/common ./packages/common
+COPY --from=build /app/packages/importer ./packages/importer
 COPY --from=build /app/packages/validation ./packages/validation
 COPY --from=build /app/packages/integrations ./packages/integrations
 COPY --from=build /app/tooling/typescript ./tooling/typescript
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -8,8 +8,7 @@
    "testing": "WORKER_PORT=9999 pnpm dev",
    "start": "node dist/index.js",
    "build": "rm -rf dist && tsdown",
-    "typecheck": "tsc --noEmit",
-    "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./src/referrers/index.ts"
+    "typecheck": "tsc --noEmit"
  },
  "dependencies": {
    "@bull-board/api": "6.13.1",
@@ -20,6 +19,7 @@
    "@openpanel/integrations": "workspace:^",
    "@openpanel/json": "workspace:*",
    "@openpanel/logger": "workspace:*",
+    "@openpanel/importer": "workspace:*",
    "@openpanel/queue": "workspace:*",
    "@openpanel/redis": "workspace:*",
    "bullmq": "^5.8.7",
@@ -38,7 +38,7 @@
    "@types/source-map-support": "^0.5.10",
    "@types/sqlstring": "^2.3.2",
    "@types/uuid": "^9.0.8",
-    "tsdown": "^0.14.2",
+    "tsdown": "0.14.2",
    "typescript": "catalog:"
  }
 }
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -5,6 +5,7 @@ import {
  type EventsQueuePayloadIncomingEvent,
  cronQueue,
  eventsGroupQueue,
+  importQueue,
  miscQueue,
  notificationQueue,
  queueLogger,
@@ -19,6 +20,7 @@ import { Worker as GroupWorker } from 'groupmq';
 import { cronJob } from './jobs/cron';
 import { eventsJob } from './jobs/events';
 import { incomingEventPure } from './jobs/events.incoming-event';
+import { importJob } from './jobs/import';
 import { miscJob } from './jobs/misc';
 import { notificationJob } from './jobs/notification';
 import { sessionsJob } from './jobs/sessions';
@@ -56,13 +58,18 @@ export async function bootWorkers() {
    workerOptions,
  );
  const miscWorker = new Worker(miscQueue.name, miscJob, workerOptions);
+  const importWorker = new Worker(importQueue.name, importJob, {
+    ...workerOptions,
+    concurrency: Number.parseInt(process.env.IMPORT_JOB_CONCURRENCY || '1', 10),
+  });

  const workers = [
    sessionsWorker,
    cronWorker,
    notificationWorker,
    miscWorker,
-    eventsGroupWorker,
+    importWorker,
+    // eventsGroupWorker,
  ];

  workers.forEach((worker) => {
@@ -148,7 +155,15 @@ export async function bootWorkers() {
  ['uncaughtException', 'unhandledRejection', 'SIGTERM', 'SIGINT'].forEach(
    (evt) => {
      process.on(evt, (code) => {
-        exitHandler(evt, code);
+        if (process.env.NODE_ENV === 'production') {
+          exitHandler(evt, code);
+        } else {
+          logger.info('Shutting down for development', {
+            event: evt,
+            code,
+          });
+          process.exit(0);
+        }
      });
    },
  );
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -5,6 +5,7 @@ import { createInitialSalts } from '@openpanel/db';
 import {
  cronQueue,
  eventsGroupQueue,
+  importQueue,
  miscQueue,
  notificationQueue,
  sessionsQueue,
@@ -31,13 +32,14 @@ async function start() {
  if (process.env.DISABLE_BULLBOARD === undefined) {
    const serverAdapter = new ExpressAdapter();
    serverAdapter.setBasePath('/');
-    ({
+    createBullBoard({
      queues: [
        new BullBoardGroupMQAdapter(eventsGroupQueue) as any,
        new BullMQAdapter(sessionsQueue),
        new BullMQAdapter(cronQueue),
        new BullMQAdapter(notificationQueue),
        new BullMQAdapter(miscQueue),
+        new BullMQAdapter(importQueue),
      ],
      serverAdapter: serverAdapter,
    });
--- a/apps/worker/src/jobs/cron.delete-projects.ts
+++ b/apps/worker/src/jobs/cron.delete-projects.ts
@@ -54,7 +54,7 @@ export async function deleteProjects(job: Job<CronQueuePayload>) {
    await ch.command({
      query,
      clickhouse_settings: {
-        lightweight_deletes_sync: 0,
+        lightweight_deletes_sync: '0',
      },
    });
  }
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -1,12 +1,15 @@
 import { logger as baseLogger } from '@/utils/logger';
-import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
 import {
  createSessionEndJob,
  createSessionStart,
  getSessionEnd,
 } from '@/utils/session-handler';
 import { isSameDomain, parsePath } from '@openpanel/common';
-import { parseUserAgent } from '@openpanel/common/server';
+import {
+  getReferrerWithQuery,
+  parseReferrer,
+  parseUserAgent,
+} from '@openpanel/common/server';
 import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
 import {
  checkNotificationRulesForEvent,
@@ -15,10 +18,9 @@ import {
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
-import { getLock } from '@openpanel/redis';
-import { DelayedError, type Job } from 'bullmq';
-import { omit } from 'ramda';
+import type { Job } from 'bullmq';
 import * as R from 'ramda';
+import { omit } from 'ramda';
 import { v4 as uuid } from 'uuid';

 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
@@ -115,9 +117,9 @@ export async function incomingEventPure(
    latitude: geo.latitude,
    path,
    origin,
-    referrer: utmReferrer?.url || referrer?.url || '',
+    referrer: referrer?.url || '',
    referrerName: utmReferrer?.name || referrer?.name || '',
-    referrerType: utmReferrer?.type || referrer?.type || '',
+    referrerType: referrer?.type || utmReferrer?.type || '',
    os: uaInfo.os,
    osVersion: uaInfo.osVersion,
    browser: uaInfo.browser,
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -99,7 +99,7 @@ describe('incomingEvent', () => {
      origin: 'https://example.com',
      referrer: '',
      referrerName: '',
-      referrerType: 'unknown',
+      referrerType: '',
      sdkName: jobData.payload.headers['openpanel-sdk-name'],
      sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
    };
@@ -207,7 +207,7 @@ describe('incomingEvent', () => {
      origin: 'https://example.com',
      referrer: '',
      referrerName: '',
-      referrerType: 'unknown',
+      referrerType: '',
      sdkName: jobData.payload.headers['openpanel-sdk-name'],
      sdkVersion: jobData.payload.headers['openpanel-sdk-version'],
    };
--- a/apps/worker/src/jobs/import.ts
+++ b/apps/worker/src/jobs/import.ts
@@ -0,0 +1,332 @@
+import {
+  type IClickhouseEvent,
+  type ImportSteps,
+  type Prisma,
+  backfillSessionsToProduction,
+  createSessionsStartEndEvents,
+  db,
+  formatClickhouseDate,
+  generateSessionIds,
+  getImportDateBounds,
+  getImportProgress,
+  insertImportBatch,
+  markImportComplete,
+  moveImportsToProduction,
+  updateImportStatus,
+} from '@openpanel/db';
+import { MixpanelProvider, UmamiProvider } from '@openpanel/importer';
+import type { ILogger } from '@openpanel/logger';
+import type { ImportQueuePayload } from '@openpanel/queue';
+import type { Job } from 'bullmq';
+import { logger } from '../utils/logger';
+
+const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10);
+
+/**
+ * Yields control back to the event loop to prevent stalled jobs
+ */
+async function yieldToEventLoop(): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, 100);
+  });
+}
+
+export async function importJob(job: Job<ImportQueuePayload>) {
+  const { importId } = job.data.payload;
+
+  const record = await db.import.findUniqueOrThrow({
+    where: { id: importId },
+    include: {
+      project: true,
+    },
+  });
+
+  const jobLogger = logger.child({
+    importId,
+    config: record.config,
+  });
+
+  type ValidStep = Exclude<ImportSteps, 'failed' | 'completed'>;
+  const steps: Record<ValidStep, number> = {
+    loading: 0,
+    generating_session_ids: 1,
+    creating_sessions: 2,
+    moving: 3,
+    backfilling_sessions: 4,
+  };
+
+  jobLogger.info('Starting import job');
+  const providerInstance = createProvider(record, jobLogger);
+
+  try {
+    // Check if this is a resume operation
+    const isNewImport = record.currentStep === null;
+
+    if (isNewImport) {
+      await updateImportStatus(jobLogger, job, importId, {
+        step: 'loading',
+      });
+    } else {
+      jobLogger.info('Resuming import from previous state', {
+        currentStep: record.currentStep,
+        currentBatch: record.currentBatch,
+      });
+    }
+
+    // Try to get a precomputed total for better progress reporting
+    const totalEvents = await providerInstance
+      .getTotalEventsCount()
+      .catch(() => -1);
+    let processedEvents = record.processedEvents;
+
+    const resumeLoadingFrom =
+      (record.currentStep === 'loading' && record.currentBatch) || undefined;
+
+    const resumeGeneratingSessionIdsFrom =
+      (record.currentStep === 'generating_session_ids' &&
+        record.currentBatch) ||
+      undefined;
+
+    const resumeCreatingSessionsFrom =
+      (record.currentStep === 'creating_sessions' && record.currentBatch) ||
+      undefined;
+
+    const resumeMovingFrom =
+      (record.currentStep === 'moving' && record.currentBatch) || undefined;
+
+    const resumeBackfillingSessionsFrom =
+      (record.currentStep === 'backfilling_sessions' && record.currentBatch) ||
+      undefined;
+
+    // Example:
+    // shouldRunStep(0) // currStep = 2 (should not run)
+    // shouldRunStep(1) // currStep = 2 (should not run)
+    // shouldRunStep(2) // currStep = 2 (should run)
+    // shouldRunStep(3) // currStep = 2 (should run)
+    const shouldRunStep = (step: ValidStep) => {
+      if (isNewImport) {
+        return true;
+      }
+
+      const stepToRunIndex = steps[step];
+      const currentStepIndex = steps[record.currentStep as ValidStep];
+      return stepToRunIndex >= currentStepIndex;
+    };
+
+    async function whileBounds(
+      from: string | undefined,
+      callback: (from: string, to: string) => Promise<void>,
+    ) {
+      const bounds = await getImportDateBounds(importId, from);
+      if (bounds.min && bounds.max) {
+        const start = new Date(bounds.min);
+        const end = new Date(bounds.max);
+        let cursor = new Date(start);
+        while (cursor < end) {
+          const next = new Date(cursor);
+          next.setDate(next.getDate() + 1);
+          await callback(
+            formatClickhouseDate(cursor, true),
+            formatClickhouseDate(next, true),
+          );
+          cursor = next;
+
+          // Yield control back to event loop after processing each day
+          await yieldToEventLoop();
+        }
+      }
+    }
+
+    // Phase 1: Fetch & Transform - Process events in batches
+    if (shouldRunStep('loading')) {
+      const eventBatch: any = [];
+      for await (const rawEvent of providerInstance.parseSource(
+        resumeLoadingFrom,
+      )) {
+        // Validate event
+        if (
+          !providerInstance.validate(
+            // @ts-expect-error
+            rawEvent,
+          )
+        ) {
+          jobLogger.warn('Skipping invalid event', { rawEvent });
+          continue;
+        }
+
+        eventBatch.push(rawEvent);
+
+        // Process batch when it reaches the batch size
+        if (eventBatch.length >= BATCH_SIZE) {
+          jobLogger.info('Processing batch', { batchSize: eventBatch.length });
+
+          const transformedEvents: IClickhouseEvent[] = eventBatch.map(
+            (
+              // @ts-expect-error
+              event,
+            ) => providerInstance!.transformEvent(event),
+          );
+
+          await insertImportBatch(transformedEvents, importId);
+
+          processedEvents += eventBatch.length;
+          eventBatch.length = 0;
+
+          const createdAt = new Date(transformedEvents[0]?.created_at || '')
+            .toISOString()
+            .split('T')[0];
+
+          await updateImportStatus(jobLogger, job, importId, {
+            step: 'loading',
+            batch: createdAt,
+            totalEvents,
+            processedEvents,
+          });
+
+          // Yield control back to event loop after processing each batch
+          await yieldToEventLoop();
+        }
+      }
+
+      // Process remaining events in the last batch
+      if (eventBatch.length > 0) {
+        const transformedEvents = eventBatch.map(
+          (
+            // @ts-expect-error
+            event,
+          ) => providerInstance!.transformEvent(event),
+        );
+
+        await insertImportBatch(transformedEvents, importId);
+
+        processedEvents += eventBatch.length;
+        eventBatch.length = 0;
+
+        const createdAt = new Date(transformedEvents[0]?.created_at || '')
+          .toISOString()
+          .split('T')[0];
+
+        await updateImportStatus(jobLogger, job, importId, {
+          step: 'loading',
+          batch: createdAt,
+        });
+
+        // Yield control back to event loop after processing final batch
+        await yieldToEventLoop();
+      }
+    }
+
+    // Phase 2: Generate session IDs if provider requires it
+    if (
+      shouldRunStep('generating_session_ids') &&
+      providerInstance.shouldGenerateSessionIds()
+    ) {
+      await whileBounds(resumeGeneratingSessionIdsFrom, async (from) => {
+        console.log('Generating session IDs', { from });
+        await generateSessionIds(importId, from);
+        await updateImportStatus(jobLogger, job, importId, {
+          step: 'generating_session_ids',
+          batch: from,
+        });
+
+        // Yield control back to event loop after processing each day
+        await yieldToEventLoop();
+      });
+
+      jobLogger.info('Session ID generation complete');
+    }
+
+    // Phase 3-5: Process in daily batches for robustness
+
+    if (shouldRunStep('creating_sessions')) {
+      await whileBounds(resumeCreatingSessionsFrom, async (from) => {
+        await createSessionsStartEndEvents(importId, from);
+        await updateImportStatus(jobLogger, job, importId, {
+          step: 'creating_sessions',
+          batch: from,
+        });
+
+        // Yield control back to event loop after processing each day
+        await yieldToEventLoop();
+      });
+    }
+
+    if (shouldRunStep('moving')) {
+      await whileBounds(resumeMovingFrom, async (from) => {
+        await moveImportsToProduction(importId, from);
+        await updateImportStatus(jobLogger, job, importId, {
+          step: 'moving',
+          batch: from,
+        });
+
+        // Yield control back to event loop after processing each day
+        await yieldToEventLoop();
+      });
+    }
+
+    if (shouldRunStep('backfilling_sessions')) {
+      await whileBounds(resumeBackfillingSessionsFrom, async (from) => {
+        await backfillSessionsToProduction(importId, from);
+        await updateImportStatus(jobLogger, job, importId, {
+          step: 'backfilling_sessions',
+          batch: from,
+        });
+
+        // Yield control back to event loop after processing each day
+        await yieldToEventLoop();
+      });
+    }
+
+    await markImportComplete(importId);
+    await updateImportStatus(jobLogger, job, importId, {
+      step: 'completed',
+    });
+    jobLogger.info('Import marked as complete');
+
+    // Get final progress
+    const finalProgress = await getImportProgress(importId);
+
+    jobLogger.info('Import job completed successfully', {
+      totalEvents: finalProgress.totalEvents,
+      insertedEvents: finalProgress.insertedEvents,
+      status: finalProgress.status,
+    });
+
+    return {
+      success: true,
+      totalEvents: finalProgress.totalEvents,
+      processedEvents: finalProgress.insertedEvents,
+    };
+  } catch (error) {
+    jobLogger.error('Import job failed', { error });
+
+    // Mark import as failed
+    try {
+      const errorMsg = error instanceof Error ? error.message : 'Unknown error';
+      await updateImportStatus(jobLogger, job, importId, {
+        step: 'failed',
+        errorMessage: errorMsg,
+      });
+      jobLogger.warn('Import marked as failed', { error: errorMsg });
+    } catch (markError) {
+      jobLogger.error('Failed to mark import as failed', { error, markError });
+    }
+
+    throw error;
+  }
+}
+
+function createProvider(
+  record: Prisma.ImportGetPayload<{ include: { project: true } }>,
+  jobLogger: ILogger,
+) {
+  const config = record.config;
+  switch (config.provider) {
+    case 'umami':
+      return new UmamiProvider(record.projectId, config, jobLogger);
+    case 'mixpanel':
+      return new MixpanelProvider(record.projectId, config, jobLogger);
+    default:
+      throw new Error(`Unknown provider: ${config.provider}`);
+  }
+}
--- a/apps/worker/src/referrers/index.ts
+++ b/apps/worker/src/referrers/index.ts
--- a/apps/worker/src/referrers/referrers.readme.md
+++ b/apps/worker/src/referrers/referrers.readme.md
@@ -1,5 +0,0 @@
-# Snowplow Referer Parser
-
-The file index.ts in this dir is generated from snowplows referer database [Snowplow Referer Parser](https://github.com/snowplow-referer-parser/referer-parser).
-
-The orginal [referers.yml](https://github.com/snowplow-referer-parser/referer-parser/blob/master/resources/referers.yml) is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.
--- a/apps/worker/src/utils/parse-referrer.test.ts
+++ b/apps/worker/src/utils/parse-referrer.test.ts
@@ -1,117 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { getReferrerWithQuery, parseReferrer } from './parse-referrer';
-
-describe('parseReferrer', () => {
-  it('should handle undefined or empty URLs', () => {
-    expect(parseReferrer(undefined)).toEqual({
-      name: '',
-      type: 'unknown',
-      url: '',
-    });
-
-    expect(parseReferrer('')).toEqual({
-      name: '',
-      type: 'unknown',
-      url: '',
-    });
-  });
-
-  it('should parse valid referrer URLs', () => {
-    expect(parseReferrer('https://google.com/search?q=test')).toEqual({
-      name: 'Google',
-      type: 'search',
-      url: 'https://google.com/search?q=test',
-    });
-  });
-
-  it('should handle www prefix in hostnames', () => {
-    expect(parseReferrer('https://www.twitter.com/user')).toEqual({
-      name: 'Twitter',
-      type: 'social',
-      url: 'https://www.twitter.com/user',
-    });
-
-    expect(parseReferrer('https://twitter.com/user')).toEqual({
-      name: 'Twitter',
-      type: 'social',
-      url: 'https://twitter.com/user',
-    });
-  });
-
-  it('should handle unknown referrers', () => {
-    expect(parseReferrer('https://unknown-site.com')).toEqual({
-      name: '',
-      type: 'unknown',
-      url: 'https://unknown-site.com',
-    });
-  });
-
-  it('should handle invalid URLs', () => {
-    expect(parseReferrer('not-a-url')).toEqual({
-      name: '',
-      type: 'unknown',
-      url: 'not-a-url',
-    });
-  });
-});
-
-describe('getReferrerWithQuery', () => {
-  it('should handle undefined or empty query', () => {
-    expect(getReferrerWithQuery(undefined)).toBeNull();
-    expect(getReferrerWithQuery({})).toBeNull();
-  });
-
-  it('should parse utm_source parameter', () => {
-    expect(getReferrerWithQuery({ utm_source: 'google' })).toEqual({
-      name: 'Google',
-      type: 'unknown',
-      url: '',
-    });
-  });
-
-  it('should parse ref parameter', () => {
-    expect(getReferrerWithQuery({ ref: 'facebook' })).toEqual({
-      name: 'Facebook',
-      type: 'social',
-      url: '',
-    });
-  });
-
-  it('should parse utm_referrer parameter', () => {
-    expect(getReferrerWithQuery({ utm_referrer: 'twitter' })).toEqual({
-      name: 'Twitter',
-      type: 'social',
-      url: '',
-    });
-  });
-
-  it('should handle case-insensitive matching', () => {
-    expect(getReferrerWithQuery({ utm_source: 'GoOgLe' })).toEqual({
-      name: 'Google',
-      type: 'unknown',
-      url: '',
-    });
-  });
-
-  it('should handle unknown sources', () => {
-    expect(getReferrerWithQuery({ utm_source: 'unknown-source' })).toEqual({
-      name: 'unknown-source',
-      type: 'unknown',
-      url: '',
-    });
-  });
-
-  it('should prioritize utm_source over ref and utm_referrer', () => {
-    expect(
-      getReferrerWithQuery({
-        utm_source: 'google',
-        ref: 'facebook',
-        utm_referrer: 'twitter',
-      }),
-    ).toEqual({
-      name: 'Google',
-      type: 'unknown',
-      url: '',
-    });
-  });
-});
--- a/apps/worker/src/utils/parse-referrer.ts
+++ b/apps/worker/src/utils/parse-referrer.ts
@@ -1,59 +0,0 @@
-import { stripTrailingSlash } from '@openpanel/common';
-
-import referrers from '../referrers';
-
-function getHostname(url: string | undefined) {
-  if (!url) {
-    return '';
-  }
-
-  try {
-    return new URL(url).hostname;
-  } catch (e) {
-    return '';
-  }
-}
-
-export function parseReferrer(url: string | undefined) {
-  const hostname = getHostname(url);
-  const match = referrers[hostname] ?? referrers[hostname.replace('www.', '')];
-
-  return {
-    name: match?.name ?? '',
-    type: match?.type ?? 'unknown',
-    url: stripTrailingSlash(url ?? ''),
-  };
-}
-
-export function getReferrerWithQuery(
-  query: Record<string, string> | undefined,
-) {
-  if (!query) {
-    return null;
-  }
-
-  const source = query.utm_source ?? query.ref ?? query.utm_referrer ?? '';
-
-  if (source === '') {
-    return null;
-  }
-
-  const match =
-    Object.values(referrers).find(
-      (referrer) => referrer.name.toLowerCase() === source.toLowerCase(),
-    ) || referrers[source];
-
-  if (match) {
-    return {
-      name: match.name,
-      type: match.type,
-      url: '',
-    };
-  }
-
-  return {
-    name: source,
-    type: 'unknown',
-    url: '',
-  };
-}