diff --git a/apps/api/package.json b/apps/api/package.json index e42cc243..9a14c7d3 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -64,7 +64,7 @@ "@types/uuid": "^10.0.0", "@types/ws": "^8.5.14", "js-yaml": "^4.1.0", - "tsdown": "^0.14.2", + "tsdown": "0.14.2", "typescript": "catalog:" } } diff --git a/apps/start/Dockerfile b/apps/start/Dockerfile index 369cd024..4853b7a6 100644 --- a/apps/start/Dockerfile +++ b/apps/start/Dockerfile @@ -29,6 +29,7 @@ COPY apps/start/package.json ./apps/start/ COPY packages/trpc/package.json packages/trpc/ COPY packages/json/package.json packages/json/ COPY packages/common/package.json packages/common/ +COPY packages/importer/package.json packages/importer/ COPY packages/payments/package.json packages/payments/ COPY packages/constants/package.json packages/constants/ COPY packages/validation/package.json packages/validation/ @@ -85,6 +86,7 @@ COPY --from=build /app/packages/trpc/package.json ./packages/trpc/ COPY --from=build /app/packages/auth/package.json ./packages/auth/ COPY --from=build /app/packages/json/package.json ./packages/json/ COPY --from=build /app/packages/common/package.json ./packages/common/ +COPY --from=build /app/packages/importer/package.json ./packages/importer/ COPY --from=build /app/packages/payments/package.json ./packages/payments/ COPY --from=build /app/packages/constants/package.json ./packages/constants/ COPY --from=build /app/packages/validation/package.json ./packages/validation/ diff --git a/apps/start/package.json b/apps/start/package.json index 4ab99a63..d5b97664 100644 --- a/apps/start/package.json +++ b/apps/start/package.json @@ -8,7 +8,7 @@ "preview": "vite preview", "deploy": "npx wrangler deploy", "cf-typegen": "wrangler types", - "build": "vite build", + "build": "pnpm with-env vite build", "serve": "vite preview", "test": "vitest run", "format": "biome format", @@ -30,6 +30,7 @@ "@openpanel/common": "workspace:^", "@openpanel/constants": "workspace:^", "@openpanel/integrations": "workspace:^", + "@openpanel/importer": "workspace:^", "@openpanel/json": "workspace:*", "@openpanel/payments": "workspace:*", "@openpanel/sdk-info": "workspace:^", diff --git a/apps/start/src/components/integrations/integration-card.tsx b/apps/start/src/components/integrations/integration-card.tsx index ab80513c..c0a35edd 100644 --- a/apps/start/src/components/integrations/integration-card.tsx +++ b/apps/start/src/components/integrations/integration-card.tsx @@ -50,12 +50,15 @@ export function IntegrationCardHeaderButtons({ export function IntegrationCardLogoImage({ src, backgroundColor, + className, }: { src: string; backgroundColor: string; + className?: string; }) { return ( []; + append: UseFieldArrayAppend; + remove: UseFieldArrayRemove; + projects: IServiceProject[]; + register: UseFormRegister; + watch: UseFormWatch; + setValue: UseFormSetValue; +} + +export function ProjectMapper({ + fields, + append, + remove, + projects, + register, + watch, + setValue, +}: ProjectMapperProps) { + return ( +
+
+ + +
+ {fields.length === 0 && ( +

+ Map source project IDs to your OpenPanel projects. If you skip mapping + all data will be imported to your current project. +

+ )} + + {fields.length > 0 && ( +
+ {fields.map((field, index) => ( +
+
+
+ + +
+
+ + +
+
+ +
+ ))} +
+ )} +
+ ); +} diff --git a/apps/start/src/components/ui/badge.tsx b/apps/start/src/components/ui/badge.tsx index e63ad31f..b8b3da2a 100644 --- a/apps/start/src/components/ui/badge.tsx +++ b/apps/start/src/components/ui/badge.tsx @@ -9,6 +9,11 @@ const badgeVariants = cva( { variants: { variant: { + success: + 'border-transparent bg-emerald-700 text-white [a&]:hover:bg-emerald-700/90', + warning: + 'border-transparent bg-yellow-500 text-white [a&]:hover:bg-yellow-500/90', + info: 'border-transparent bg-blue-500 text-white [a&]:hover:bg-blue-500/90', default: 'border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90', secondary: diff --git a/apps/start/src/modals/add-import.tsx b/apps/start/src/modals/add-import.tsx new file mode 100644 index 00000000..03a77e35 --- /dev/null +++ b/apps/start/src/modals/add-import.tsx @@ -0,0 +1,282 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { toast } from 'sonner'; + +import { InputWithLabel, WithLabel } from '@/components/forms/input-with-label'; +import { ProjectMapper } from '@/components/project-mapper'; +import { Button } from '@/components/ui/button'; +import { useAppParams } from '@/hooks/use-app-params'; +import { useTRPC } from '@/integrations/trpc/react'; +import { cn } from '@/lib/utils'; +import { zodResolver } from '@hookform/resolvers/zod'; +import type { + IImportConfig, + IMixpanelImportConfig, + IUmamiImportConfig, +} from '@openpanel/validation'; +import { + zMixpanelImportConfig, + zUmamiImportConfig, +} from '@openpanel/validation'; +import { format } from 'date-fns'; +import { CalendarIcon } from 'lucide-react'; +import { useFieldArray, useForm } from 'react-hook-form'; +import type { z } from 'zod'; +import { popModal, pushModal } from '.'; +import { ModalContent, ModalHeader } from './Modal/Container'; + +type Provider = 'umami' | 'plausible' | 'mixpanel'; + +interface AddImportProps { + provider: Provider; + name: string; + types: ('file' | 'api')[]; +} + +type UmamiFormData = z.infer; +type MixpanelFormData = z.infer; + +interface UmamiImportProps { + onSubmit: (config: IUmamiImportConfig) => void; + isPending: boolean; + organizationId: string; +} + +function UmamiImport({ + onSubmit, + isPending, + organizationId, +}: UmamiImportProps) { + const trpc = useTRPC(); + const { data: projects = [] } = useQuery( + trpc.project.list.queryOptions({ + organizationId, + }), + ); + + const form = useForm({ + resolver: zodResolver(zUmamiImportConfig), + defaultValues: { + provider: 'umami', + type: 'file', + fileUrl: '', + projectMapper: [], + }, + }); + + const { fields, append, remove } = useFieldArray({ + control: form.control, + name: 'projectMapper', + }); + + const handleSubmit = form.handleSubmit((data) => { + onSubmit(data); + }); + + return ( +
+
+ + + +
+ +
+ + +
+
+ ); +} + +interface MixpanelImportProps { + onSubmit: (config: IMixpanelImportConfig) => void; + isPending: boolean; + organizationId: string; +} + +function MixpanelImport({ + onSubmit, + isPending, + organizationId, +}: MixpanelImportProps) { + const trpc = useTRPC(); + const form = useForm({ + resolver: zodResolver(zMixpanelImportConfig), + defaultValues: { + provider: 'mixpanel', + type: 'api', + serviceAccount: '', + serviceSecret: '', + projectId: '', + from: '', + to: '', + }, + }); + + const handleDateRangeSelect = () => { + pushModal('DateRangerPicker', { + startDate: form.getValues('from') + ? new Date(form.getValues('from')) + : undefined, + endDate: form.getValues('to') + ? new Date(form.getValues('to')) + : undefined, + onChange: ({ startDate, endDate }) => { + form.setValue('from', format(startDate, 'yyyy-MM-dd')); + form.setValue('to', format(endDate, 'yyyy-MM-dd')); + form.trigger('from'); + form.trigger('to'); + }, + }); + }; + + const handleSubmit = form.handleSubmit((data) => { + onSubmit(data); + }); + + return ( +
+
+ + + + + + + + + + + +
+ +
+ + +
+
+ ); +} + +export default function AddImport({ provider, name }: AddImportProps) { + const { projectId, organizationId } = useAppParams(); + const trpc = useTRPC(); + const queryClient = useQueryClient(); + + const createImport = useMutation( + trpc.import.create.mutationOptions({ + onSuccess() { + toast.success('Import started', { + description: 'Your data import has been queued for processing.', + }); + popModal(); + queryClient.invalidateQueries(trpc.import.list.pathFilter()); + }, + onError: (error) => { + toast.error('Import failed', { + description: error.message, + }); + }, + }), + ); + + const handleImportSubmit = (config: IImportConfig) => { + createImport.mutate({ + projectId, + provider: config.provider, + config, + }); + }; + + return ( + + + + {provider === 'umami' && ( + + )} + + {provider === 'mixpanel' && ( + + )} + + ); +} diff --git a/apps/start/src/modals/date-ranger-picker.tsx b/apps/start/src/modals/date-ranger-picker.tsx index 7c114229..277fa163 100644 --- a/apps/start/src/modals/date-ranger-picker.tsx +++ b/apps/start/src/modals/date-ranger-picker.tsx @@ -27,6 +27,7 @@ export default function DateRangerPicker({ return ( AppOrganizationIdProjectIdEventsTabsRoute, } as any) +const AppOrganizationIdProjectIdSettingsTabsImportsRoute = + AppOrganizationIdProjectIdSettingsTabsImportsRouteImport.update({ + id: '/imports', + path: '/imports', + getParentRoute: () => AppOrganizationIdProjectIdSettingsTabsRoute, + } as any) const AppOrganizationIdProjectIdSettingsTabsEventsRoute = AppOrganizationIdProjectIdSettingsTabsEventsRouteImport.update({ id: '/events', @@ -515,6 +522,7 @@ export interface FileRoutesByFullPath { '/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute '/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute '/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute '/$organizationId/$projectId/events/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute '/$organizationId/$projectId/notifications/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute '/$organizationId/$projectId/profiles/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute @@ -569,6 +577,7 @@ export interface FileRoutesByTo { '/$organizationId/$projectId/settings/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute '/$organizationId/$projectId/settings/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute '/$organizationId/$projectId/settings/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + '/$organizationId/$projectId/settings/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute '/$organizationId/$projectId/profiles/$profileId/events': typeof AppOrganizationIdProjectIdProfilesProfileIdTabsEventsRoute } export interface FileRoutesById { @@ -633,6 +642,7 @@ export interface FileRoutesById { '/_app/$organizationId/$projectId_/settings/_tabs/clients': typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute '/_app/$organizationId/$projectId_/settings/_tabs/details': typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute '/_app/$organizationId/$projectId_/settings/_tabs/events': typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + '/_app/$organizationId/$projectId_/settings/_tabs/imports': typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute '/_app/$organizationId/$projectId_/events/_tabs/': typeof AppOrganizationIdProjectIdEventsTabsIndexRoute '/_app/$organizationId/$projectId_/notifications/_tabs/': typeof AppOrganizationIdProjectIdNotificationsTabsIndexRoute '/_app/$organizationId/$projectId_/profiles/_tabs/': typeof AppOrganizationIdProjectIdProfilesTabsIndexRoute @@ -692,6 +702,7 @@ export interface FileRouteTypes { | '/$organizationId/$projectId/settings/clients' | '/$organizationId/$projectId/settings/details' | '/$organizationId/$projectId/settings/events' + | '/$organizationId/$projectId/settings/imports' | '/$organizationId/$projectId/events/' | '/$organizationId/$projectId/notifications/' | '/$organizationId/$projectId/profiles/' @@ -746,6 +757,7 @@ export interface FileRouteTypes { | '/$organizationId/$projectId/settings/clients' | '/$organizationId/$projectId/settings/details' | '/$organizationId/$projectId/settings/events' + | '/$organizationId/$projectId/settings/imports' | '/$organizationId/$projectId/profiles/$profileId/events' id: | '__root__' @@ -809,6 +821,7 @@ export interface FileRouteTypes { | '/_app/$organizationId/$projectId_/settings/_tabs/clients' | '/_app/$organizationId/$projectId_/settings/_tabs/details' | '/_app/$organizationId/$projectId_/settings/_tabs/events' + | '/_app/$organizationId/$projectId_/settings/_tabs/imports' | '/_app/$organizationId/$projectId_/events/_tabs/' | '/_app/$organizationId/$projectId_/notifications/_tabs/' | '/_app/$organizationId/$projectId_/profiles/_tabs/' @@ -1194,6 +1207,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AppOrganizationIdProjectIdEventsTabsIndexRouteImport parentRoute: typeof AppOrganizationIdProjectIdEventsTabsRoute } + '/_app/$organizationId/$projectId_/settings/_tabs/imports': { + id: '/_app/$organizationId/$projectId_/settings/_tabs/imports' + path: '/imports' + fullPath: '/$organizationId/$projectId/settings/imports' + preLoaderRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRouteImport + parentRoute: typeof AppOrganizationIdProjectIdSettingsTabsRoute + } '/_app/$organizationId/$projectId_/settings/_tabs/events': { id: '/_app/$organizationId/$projectId_/settings/_tabs/events' path: '/events' @@ -1521,6 +1541,7 @@ interface AppOrganizationIdProjectIdSettingsTabsRouteChildren { AppOrganizationIdProjectIdSettingsTabsClientsRoute: typeof AppOrganizationIdProjectIdSettingsTabsClientsRoute AppOrganizationIdProjectIdSettingsTabsDetailsRoute: typeof AppOrganizationIdProjectIdSettingsTabsDetailsRoute AppOrganizationIdProjectIdSettingsTabsEventsRoute: typeof AppOrganizationIdProjectIdSettingsTabsEventsRoute + AppOrganizationIdProjectIdSettingsTabsImportsRoute: typeof AppOrganizationIdProjectIdSettingsTabsImportsRoute AppOrganizationIdProjectIdSettingsTabsIndexRoute: typeof AppOrganizationIdProjectIdSettingsTabsIndexRoute } @@ -1532,6 +1553,8 @@ const AppOrganizationIdProjectIdSettingsTabsRouteChildren: AppOrganizationIdProj AppOrganizationIdProjectIdSettingsTabsDetailsRoute, AppOrganizationIdProjectIdSettingsTabsEventsRoute: AppOrganizationIdProjectIdSettingsTabsEventsRoute, + AppOrganizationIdProjectIdSettingsTabsImportsRoute: + AppOrganizationIdProjectIdSettingsTabsImportsRoute, AppOrganizationIdProjectIdSettingsTabsIndexRoute: AppOrganizationIdProjectIdSettingsTabsIndexRoute, } diff --git a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx new file mode 100644 index 00000000..123e7875 --- /dev/null +++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.imports.tsx @@ -0,0 +1,303 @@ +import { FullPageEmptyState } from '@/components/full-page-empty-state'; +import { + IntegrationCard, + IntegrationCardFooter, + IntegrationCardLogoImage, +} from '@/components/integrations/integration-card'; +import { Skeleton } from '@/components/skeleton'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { Tooltiper } from '@/components/ui/tooltip'; +import { useAppParams } from '@/hooks/use-app-params'; +import { useTRPC } from '@/integrations/trpc/react'; +import { pushModal } from '@/modals'; +import { IMPORT_PROVIDERS } from '@openpanel/importer/providers'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { createFileRoute } from '@tanstack/react-router'; +import { formatDistanceToNow } from 'date-fns'; +import { + CheckCircleIcon, + Download, + InfoIcon, + Loader2Icon, + XCircleIcon, +} from 'lucide-react'; +import { toast } from 'sonner'; + +export const Route = createFileRoute( + '/_app/$organizationId/$projectId_/settings/_tabs/imports', +)({ + component: ImportsSettings, +}); + +function ImportsSettings() { + const { projectId } = useAppParams(); + const trpc = useTRPC(); + const queryClient = useQueryClient(); + + const importsQuery = useQuery( + trpc.import.list.queryOptions( + { projectId }, + { + refetchInterval: 5000, + }, + ), + ); + const imports = importsQuery.data ?? []; + + const deleteImport = useMutation( + trpc.import.delete.mutationOptions({ + onSuccess: () => { + toast.success('Import deleted', { + description: 'The import has been successfully deleted.', + }); + queryClient.invalidateQueries(trpc.import.list.pathFilter()); + }, + }), + ); + + const retryImport = useMutation( + trpc.import.retry.mutationOptions({ + onSuccess: () => { + toast.success('Import retried', { + description: 'The import has been queued for processing again.', + }); + queryClient.invalidateQueries(trpc.import.list.pathFilter()); + }, + }), + ); + + const handleProviderSelect = ( + provider: (typeof IMPORT_PROVIDERS)[number], + ) => { + pushModal('AddImport', { + provider: provider.id, + name: provider.name, + types: provider.types, + }); + }; + + const getStatusBadge = (status: string, errorMessage: string | null) => { + const variants: Record = { + pending: 'secondary', + processing: 'default', + completed: 'success', + failed: 'destructive', + }; + const icons: Record = { + pending: , + processing: , + completed: , + failed: , + }; + + if (status === 'failed') { + return ( + + + {icons[status] || null} + {status} + + + ); + } + + return ( + + {icons[status] || null} + {status} + + ); + }; + + return ( +
+
+
+ {IMPORT_PROVIDERS.map((provider) => ( + + } + name={provider.name} + description={provider.description} + > + + + + + ))} +
+
+ +
+

Import History

+ +
+ + + + Provider + Created + Status + Progress + Config + Actions + + + + {!importsQuery.isLoading && imports.length === 0 && ( + + + + + + )} + {importsQuery.isLoading && + [1, 2, 3, 4].map((index) => ( + + + + + + + + + + + + + + + + + + + + + ))} + {imports.map((imp) => ( + + +
+
{imp.config.provider}
+ + {imp.config.type} + +
+ + + {formatDistanceToNow(new Date(imp.createdAt), { + addSuffix: true, + })} + + +
+ {getStatusBadge(imp.status, imp.errorMessage)} + {imp.statusMessage && ( +
+ {imp.statusMessage} +
+ )} +
+ + + {imp.totalEvents > 0 ? ( +
+
+ {imp.processedEvents.toLocaleString()} + {' / '} + + {imp.totalEvents.toLocaleString()}{' '} + + +
+ {imp.status === 'processing' && ( +
+
+
+ )} +
+ ) : imp.totalEvents === -1 ? ( +
+ {imp.processedEvents.toLocaleString()} + {' / '} + N/A +
+ ) : ( + '-' + )} + + + + + {JSON.stringify(imp.config, null, 2)} + + } + tooltipClassName="max-w-xs" + > + Config + + + + {imp.status === 'failed' && ( + + )} + + + + ))} + +
+
+
+
+ ); +} diff --git a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx index 39dc0cb3..0c7f217a 100644 --- a/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx +++ b/apps/start/src/routes/_app.$organizationId.$projectId_.settings._tabs.tsx @@ -42,6 +42,7 @@ function ProjectDashboard() { { id: 'details', label: 'Details' }, { id: 'events', label: 'Events' }, { id: 'clients', label: 'Clients' }, + { id: 'imports', label: 'Imports' }, ]; const handleTabChange = (tabId: string) => { diff --git a/apps/start/src/utils/math.ts b/apps/start/src/utils/math.ts index e75c1964..e41d4e1f 100644 --- a/apps/start/src/utils/math.ts +++ b/apps/start/src/utils/math.ts @@ -1 +1 @@ -export * from '@openpanel/common/src/math'; +export * from '@openpanel/common'; diff --git a/apps/start/src/utils/op.ts b/apps/start/src/utils/op.ts index 3e9cfee1..0ba0fa95 100644 --- a/apps/start/src/utils/op.ts +++ b/apps/start/src/utils/op.ts @@ -1,8 +1,20 @@ import { OpenPanel } from '@openpanel/web'; -export const op = new OpenPanel({ - clientId: import.meta.env.VITE_OP_CLIENT_ID, - trackScreenViews: true, - trackOutgoingLinks: true, - trackAttributes: true, -}); +const clientId = import.meta.env.VITE_OP_CLIENT_ID; + +const createOpInstance = () => { + if (!clientId || clientId === 'undefined') { + return new Proxy({} as OpenPanel, { + get: () => () => {}, + }); + } + + return new OpenPanel({ + clientId, + trackScreenViews: true, + trackOutgoingLinks: true, + trackAttributes: true, + }); +}; + +export const op = createOpInstance(); diff --git a/apps/start/src/utils/slug.ts b/apps/start/src/utils/slug.ts index ebdad3b9..e41d4e1f 100644 --- a/apps/start/src/utils/slug.ts +++ b/apps/start/src/utils/slug.ts @@ -1 +1 @@ -export * from '@openpanel/common/src/slug'; +export * from '@openpanel/common'; diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 7f953f59..b816beef 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -35,6 +35,7 @@ COPY packages/redis/package.json ./packages/redis/ COPY packages/queue/package.json ./packages/queue/ COPY packages/logger/package.json ./packages/logger/ COPY packages/common/package.json ./packages/common/ +COPY packages/importer/package.json ./packages/importer/ COPY packages/constants/package.json ./packages/constants/ COPY packages/validation/package.json ./packages/validation/ COPY packages/integrations/package.json packages/integrations/ @@ -80,9 +81,10 @@ COPY --from=build /app/packages/geo ./packages/geo COPY --from=build /app/packages/json ./packages/json COPY --from=build /app/packages/email ./packages/email COPY --from=build /app/packages/redis ./packages/redis -COPY --from=build /app/packages/logger ./packages/logger COPY --from=build /app/packages/queue ./packages/queue +COPY --from=build /app/packages/logger ./packages/logger COPY --from=build /app/packages/common ./packages/common +COPY --from=build /app/packages/importer ./packages/importer COPY --from=build /app/packages/validation ./packages/validation COPY --from=build /app/packages/integrations ./packages/integrations COPY --from=build /app/tooling/typescript ./tooling/typescript diff --git a/apps/worker/package.json b/apps/worker/package.json index c9b54878..fc1b2d2e 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -8,8 +8,7 @@ "testing": "WORKER_PORT=9999 pnpm dev", "start": "node dist/index.js", "build": "rm -rf dist && tsdown", - "typecheck": "tsc --noEmit", - "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./src/referrers/index.ts" + "typecheck": "tsc --noEmit" }, "dependencies": { "@bull-board/api": "6.13.1", @@ -20,6 +19,7 @@ "@openpanel/integrations": "workspace:^", "@openpanel/json": "workspace:*", "@openpanel/logger": "workspace:*", + "@openpanel/importer": "workspace:*", "@openpanel/queue": "workspace:*", "@openpanel/redis": "workspace:*", "bullmq": "^5.8.7", @@ -38,7 +38,7 @@ "@types/source-map-support": "^0.5.10", "@types/sqlstring": "^2.3.2", "@types/uuid": "^9.0.8", - "tsdown": "^0.14.2", + "tsdown": "0.14.2", "typescript": "catalog:" } } diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts index d99e14b3..d64bcf10 100644 --- a/apps/worker/src/boot-workers.ts +++ b/apps/worker/src/boot-workers.ts @@ -5,6 +5,7 @@ import { type EventsQueuePayloadIncomingEvent, cronQueue, eventsGroupQueue, + importQueue, miscQueue, notificationQueue, queueLogger, @@ -19,6 +20,7 @@ import { Worker as GroupWorker } from 'groupmq'; import { cronJob } from './jobs/cron'; import { eventsJob } from './jobs/events'; import { incomingEventPure } from './jobs/events.incoming-event'; +import { importJob } from './jobs/import'; import { miscJob } from './jobs/misc'; import { notificationJob } from './jobs/notification'; import { sessionsJob } from './jobs/sessions'; @@ -56,13 +58,18 @@ export async function bootWorkers() { workerOptions, ); const miscWorker = new Worker(miscQueue.name, miscJob, workerOptions); + const importWorker = new Worker(importQueue.name, importJob, { + ...workerOptions, + concurrency: Number.parseInt(process.env.IMPORT_JOB_CONCURRENCY || '1', 10), + }); const workers = [ sessionsWorker, cronWorker, notificationWorker, miscWorker, - eventsGroupWorker, + importWorker, + // eventsGroupWorker, ]; workers.forEach((worker) => { @@ -148,7 +155,15 @@ export async function bootWorkers() { ['uncaughtException', 'unhandledRejection', 'SIGTERM', 'SIGINT'].forEach( (evt) => { process.on(evt, (code) => { - exitHandler(evt, code); + if (process.env.NODE_ENV === 'production') { + exitHandler(evt, code); + } else { + logger.info('Shutting down for development', { + event: evt, + code, + }); + process.exit(0); + } }); }, ); diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index bb7793d7..74bb3cb6 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -5,6 +5,7 @@ import { createInitialSalts } from '@openpanel/db'; import { cronQueue, eventsGroupQueue, + importQueue, miscQueue, notificationQueue, sessionsQueue, @@ -31,13 +32,14 @@ async function start() { if (process.env.DISABLE_BULLBOARD === undefined) { const serverAdapter = new ExpressAdapter(); serverAdapter.setBasePath('/'); - ({ + createBullBoard({ queues: [ new BullBoardGroupMQAdapter(eventsGroupQueue) as any, new BullMQAdapter(sessionsQueue), new BullMQAdapter(cronQueue), new BullMQAdapter(notificationQueue), new BullMQAdapter(miscQueue), + new BullMQAdapter(importQueue), ], serverAdapter: serverAdapter, }); diff --git a/apps/worker/src/jobs/cron.delete-projects.ts b/apps/worker/src/jobs/cron.delete-projects.ts index abb8f684..eab4f84c 100644 --- a/apps/worker/src/jobs/cron.delete-projects.ts +++ b/apps/worker/src/jobs/cron.delete-projects.ts @@ -54,7 +54,7 @@ export async function deleteProjects(job: Job) { await ch.command({ query, clickhouse_settings: { - lightweight_deletes_sync: 0, + lightweight_deletes_sync: '0', }, }); } diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts index ecd28a55..bb3f3743 100644 --- a/apps/worker/src/jobs/events.incoming-event.ts +++ b/apps/worker/src/jobs/events.incoming-event.ts @@ -1,12 +1,15 @@ import { logger as baseLogger } from '@/utils/logger'; -import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer'; import { createSessionEndJob, createSessionStart, getSessionEnd, } from '@/utils/session-handler'; import { isSameDomain, parsePath } from '@openpanel/common'; -import { parseUserAgent } from '@openpanel/common/server'; +import { + getReferrerWithQuery, + parseReferrer, + parseUserAgent, +} from '@openpanel/common/server'; import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db'; import { checkNotificationRulesForEvent, @@ -15,10 +18,9 @@ import { } from '@openpanel/db'; import type { ILogger } from '@openpanel/logger'; import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue'; -import { getLock } from '@openpanel/redis'; -import { DelayedError, type Job } from 'bullmq'; -import { omit } from 'ramda'; +import type { Job } from 'bullmq'; import * as R from 'ramda'; +import { omit } from 'ramda'; import { v4 as uuid } from 'uuid'; const GLOBAL_PROPERTIES = ['__path', '__referrer']; @@ -115,9 +117,9 @@ export async function incomingEventPure( latitude: geo.latitude, path, origin, - referrer: utmReferrer?.url || referrer?.url || '', + referrer: referrer?.url || '', referrerName: utmReferrer?.name || referrer?.name || '', - referrerType: utmReferrer?.type || referrer?.type || '', + referrerType: referrer?.type || utmReferrer?.type || '', os: uaInfo.os, osVersion: uaInfo.osVersion, browser: uaInfo.browser, diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.ts index 292453d4..d5d9dd0c 100644 --- a/apps/worker/src/jobs/events.incoming-events.test.ts +++ b/apps/worker/src/jobs/events.incoming-events.test.ts @@ -99,7 +99,7 @@ describe('incomingEvent', () => { origin: 'https://example.com', referrer: '', referrerName: '', - referrerType: 'unknown', + referrerType: '', sdkName: jobData.payload.headers['openpanel-sdk-name'], sdkVersion: jobData.payload.headers['openpanel-sdk-version'], }; @@ -207,7 +207,7 @@ describe('incomingEvent', () => { origin: 'https://example.com', referrer: '', referrerName: '', - referrerType: 'unknown', + referrerType: '', sdkName: jobData.payload.headers['openpanel-sdk-name'], sdkVersion: jobData.payload.headers['openpanel-sdk-version'], }; diff --git a/apps/worker/src/jobs/import.ts b/apps/worker/src/jobs/import.ts new file mode 100644 index 00000000..0346d583 --- /dev/null +++ b/apps/worker/src/jobs/import.ts @@ -0,0 +1,332 @@ +import { + type IClickhouseEvent, + type ImportSteps, + type Prisma, + backfillSessionsToProduction, + createSessionsStartEndEvents, + db, + formatClickhouseDate, + generateSessionIds, + getImportDateBounds, + getImportProgress, + insertImportBatch, + markImportComplete, + moveImportsToProduction, + updateImportStatus, +} from '@openpanel/db'; +import { MixpanelProvider, UmamiProvider } from '@openpanel/importer'; +import type { ILogger } from '@openpanel/logger'; +import type { ImportQueuePayload } from '@openpanel/queue'; +import type { Job } from 'bullmq'; +import { logger } from '../utils/logger'; + +const BATCH_SIZE = Number.parseInt(process.env.IMPORT_BATCH_SIZE || '5000', 10); + +/** + * Yields control back to the event loop to prevent stalled jobs + */ +async function yieldToEventLoop(): Promise { + return new Promise((resolve) => { + setTimeout(resolve, 100); + }); +} + +export async function importJob(job: Job) { + const { importId } = job.data.payload; + + const record = await db.import.findUniqueOrThrow({ + where: { id: importId }, + include: { + project: true, + }, + }); + + const jobLogger = logger.child({ + importId, + config: record.config, + }); + + type ValidStep = Exclude; + const steps: Record = { + loading: 0, + generating_session_ids: 1, + creating_sessions: 2, + moving: 3, + backfilling_sessions: 4, + }; + + jobLogger.info('Starting import job'); + const providerInstance = createProvider(record, jobLogger); + + try { + // Check if this is a resume operation + const isNewImport = record.currentStep === null; + + if (isNewImport) { + await updateImportStatus(jobLogger, job, importId, { + step: 'loading', + }); + } else { + jobLogger.info('Resuming import from previous state', { + currentStep: record.currentStep, + currentBatch: record.currentBatch, + }); + } + + // Try to get a precomputed total for better progress reporting + const totalEvents = await providerInstance + .getTotalEventsCount() + .catch(() => -1); + let processedEvents = record.processedEvents; + + const resumeLoadingFrom = + (record.currentStep === 'loading' && record.currentBatch) || undefined; + + const resumeGeneratingSessionIdsFrom = + (record.currentStep === 'generating_session_ids' && + record.currentBatch) || + undefined; + + const resumeCreatingSessionsFrom = + (record.currentStep === 'creating_sessions' && record.currentBatch) || + undefined; + + const resumeMovingFrom = + (record.currentStep === 'moving' && record.currentBatch) || undefined; + + const resumeBackfillingSessionsFrom = + (record.currentStep === 'backfilling_sessions' && record.currentBatch) || + undefined; + + // Example: + // shouldRunStep(0) // currStep = 2 (should not run) + // shouldRunStep(1) // currStep = 2 (should not run) + // shouldRunStep(2) // currStep = 2 (should run) + // shouldRunStep(3) // currStep = 2 (should run) + const shouldRunStep = (step: ValidStep) => { + if (isNewImport) { + return true; + } + + const stepToRunIndex = steps[step]; + const currentStepIndex = steps[record.currentStep as ValidStep]; + return stepToRunIndex >= currentStepIndex; + }; + + async function whileBounds( + from: string | undefined, + callback: (from: string, to: string) => Promise, + ) { + const bounds = await getImportDateBounds(importId, from); + if (bounds.min && bounds.max) { + const start = new Date(bounds.min); + const end = new Date(bounds.max); + let cursor = new Date(start); + while (cursor < end) { + const next = new Date(cursor); + next.setDate(next.getDate() + 1); + await callback( + formatClickhouseDate(cursor, true), + formatClickhouseDate(next, true), + ); + cursor = next; + + // Yield control back to event loop after processing each day + await yieldToEventLoop(); + } + } + } + + // Phase 1: Fetch & Transform - Process events in batches + if (shouldRunStep('loading')) { + const eventBatch: any = []; + for await (const rawEvent of providerInstance.parseSource( + resumeLoadingFrom, + )) { + // Validate event + if ( + !providerInstance.validate( + // @ts-expect-error + rawEvent, + ) + ) { + jobLogger.warn('Skipping invalid event', { rawEvent }); + continue; + } + + eventBatch.push(rawEvent); + + // Process batch when it reaches the batch size + if (eventBatch.length >= BATCH_SIZE) { + jobLogger.info('Processing batch', { batchSize: eventBatch.length }); + + const transformedEvents: IClickhouseEvent[] = eventBatch.map( + ( + // @ts-expect-error + event, + ) => providerInstance!.transformEvent(event), + ); + + await insertImportBatch(transformedEvents, importId); + + processedEvents += eventBatch.length; + eventBatch.length = 0; + + const createdAt = new Date(transformedEvents[0]?.created_at || '') + .toISOString() + .split('T')[0]; + + await updateImportStatus(jobLogger, job, importId, { + step: 'loading', + batch: createdAt, + totalEvents, + processedEvents, + }); + + // Yield control back to event loop after processing each batch + await yieldToEventLoop(); + } + } + + // Process remaining events in the last batch + if (eventBatch.length > 0) { + const transformedEvents = eventBatch.map( + ( + // @ts-expect-error + event, + ) => providerInstance!.transformEvent(event), + ); + + await insertImportBatch(transformedEvents, importId); + + processedEvents += eventBatch.length; + eventBatch.length = 0; + + const createdAt = new Date(transformedEvents[0]?.created_at || '') + .toISOString() + .split('T')[0]; + + await updateImportStatus(jobLogger, job, importId, { + step: 'loading', + batch: createdAt, + }); + + // Yield control back to event loop after processing final batch + await yieldToEventLoop(); + } + } + + // Phase 2: Generate session IDs if provider requires it + if ( + shouldRunStep('generating_session_ids') && + providerInstance.shouldGenerateSessionIds() + ) { + await whileBounds(resumeGeneratingSessionIdsFrom, async (from) => { + console.log('Generating session IDs', { from }); + await generateSessionIds(importId, from); + await updateImportStatus(jobLogger, job, importId, { + step: 'generating_session_ids', + batch: from, + }); + + // Yield control back to event loop after processing each day + await yieldToEventLoop(); + }); + + jobLogger.info('Session ID generation complete'); + } + + // Phase 3-5: Process in daily batches for robustness + + if (shouldRunStep('creating_sessions')) { + await whileBounds(resumeCreatingSessionsFrom, async (from) => { + await createSessionsStartEndEvents(importId, from); + await updateImportStatus(jobLogger, job, importId, { + step: 'creating_sessions', + batch: from, + }); + + // Yield control back to event loop after processing each day + await yieldToEventLoop(); + }); + } + + if (shouldRunStep('moving')) { + await whileBounds(resumeMovingFrom, async (from) => { + await moveImportsToProduction(importId, from); + await updateImportStatus(jobLogger, job, importId, { + step: 'moving', + batch: from, + }); + + // Yield control back to event loop after processing each day + await yieldToEventLoop(); + }); + } + + if (shouldRunStep('backfilling_sessions')) { + await whileBounds(resumeBackfillingSessionsFrom, async (from) => { + await backfillSessionsToProduction(importId, from); + await updateImportStatus(jobLogger, job, importId, { + step: 'backfilling_sessions', + batch: from, + }); + + // Yield control back to event loop after processing each day + await yieldToEventLoop(); + }); + } + + await markImportComplete(importId); + await updateImportStatus(jobLogger, job, importId, { + step: 'completed', + }); + jobLogger.info('Import marked as complete'); + + // Get final progress + const finalProgress = await getImportProgress(importId); + + jobLogger.info('Import job completed successfully', { + totalEvents: finalProgress.totalEvents, + insertedEvents: finalProgress.insertedEvents, + status: finalProgress.status, + }); + + return { + success: true, + totalEvents: finalProgress.totalEvents, + processedEvents: finalProgress.insertedEvents, + }; + } catch (error) { + jobLogger.error('Import job failed', { error }); + + // Mark import as failed + try { + const errorMsg = error instanceof Error ? error.message : 'Unknown error'; + await updateImportStatus(jobLogger, job, importId, { + step: 'failed', + errorMessage: errorMsg, + }); + jobLogger.warn('Import marked as failed', { error: errorMsg }); + } catch (markError) { + jobLogger.error('Failed to mark import as failed', { error, markError }); + } + + throw error; + } +} + +function createProvider( + record: Prisma.ImportGetPayload<{ include: { project: true } }>, + jobLogger: ILogger, +) { + const config = record.config; + switch (config.provider) { + case 'umami': + return new UmamiProvider(record.projectId, config, jobLogger); + case 'mixpanel': + return new MixpanelProvider(record.projectId, config, jobLogger); + default: + throw new Error(`Unknown provider: ${config.provider}`); + } +} diff --git a/package.json b/package.json index 0860945f..c2985585 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts": { "test": "vitest run", "gen:bots": "pnpm -r --filter api gen:bots", - "gen:referrers": "pnpm -r --filter worker gen:referrers", + "gen:referrers": "pnpm -r --filter common gen:referrers", "dock:up": "docker compose up -d", "dock:down": "docker compose down", "dock:ch": "docker compose exec -it op-ch clickhouse-client -d openpanel", diff --git a/packages/cli/package.json b/packages/cli/package.json deleted file mode 100644 index d352ce72..00000000 --- a/packages/cli/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@openpanel/cli", - "version": "0.0.1-beta", - "type": "module", - "module": "index.ts", - "bin": { - "openpanel": "dist/bin/cli.js" - }, - "scripts": { - "build": "rm -rf dist && tsup", - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "@openpanel/common": "workspace:*", - "arg": "^5.0.2", - "glob": "^10.4.3", - "inquirer": "^9.3.5", - "p-limit": "^6.1.0", - "progress": "^2.0.3", - "ramda": "^0.29.1", - "zod": "catalog:" - }, - "devDependencies": { - "@openpanel/db": "workspace:^", - "@openpanel/sdk": "workspace:*", - "@openpanel/tsconfig": "workspace:*", - "@types/node": "catalog:", - "@types/progress": "^2.0.7", - "@types/ramda": "^0.30.1", - "tsup": "^7.2.0", - "typescript": "catalog:" - } -} diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts deleted file mode 100644 index 872aa731..00000000 --- a/packages/cli/src/cli.ts +++ /dev/null @@ -1,24 +0,0 @@ -import arg from 'arg'; - -import importer from './importer'; - -function cli() { - const args = arg( - { - '--help': Boolean, - }, - { - permissive: true, - }, - ); - - const [command] = args._; - - switch (command) { - case 'import': { - return importer(); - } - } -} - -cli(); diff --git a/packages/cli/src/importer/importer.ts b/packages/cli/src/importer/importer.ts deleted file mode 100644 index 27be17af..00000000 --- a/packages/cli/src/importer/importer.ts +++ /dev/null @@ -1,467 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import readline from 'node:readline'; -import zlib from 'node:zlib'; -import Progress from 'progress'; -import { assocPath, prop, uniqBy } from 'ramda'; - -import { isSameDomain, parsePath } from '@openpanel/common'; -import type { IImportedEvent } from '@openpanel/db'; - -const BATCH_SIZE = 30_000; -const SLEEP_TIME = 20; -const MAX_CONCURRENT_REQUESTS = 8; - -type IMixpanelEvent = { - event: string; - properties: { - [key: string]: unknown; - time: number; - $current_url?: string; - distinct_id?: string; - $device_id?: string; - country_code?: string; - $region?: string; - $city?: string; - $os?: string; - $browser?: string; - $browser_version?: string; - $initial_referrer?: string; - $search_engine?: string; - }; -}; - -function stripMixpanelProperties(obj: Record) { - return Object.fromEntries( - Object.entries(obj).filter( - ([key]) => - !key.match(/^(\$|mp_)/) && !['time', 'distinct_id'].includes(key), - ), - ); -} - -async function* parseJsonStream( - fileStream: fs.ReadStream, -): AsyncGenerator { - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Number.POSITIVE_INFINITY, - }); - - let buffer = ''; - let bracketCount = 0; - - for await (const line of rl) { - buffer += line; - bracketCount += - (line.match(/{/g) || []).length - (line.match(/}/g) || []).length; - - if (bracketCount === 0 && buffer.trim()) { - try { - const json = JSON.parse(buffer); - yield json; - } catch (error) { - console.log('Warning: Failed to parse JSON'); - console.log('Buffer:', buffer); - } - buffer = ''; - } - } - - if (buffer.trim()) { - try { - const json = JSON.parse(buffer); - yield json; - } catch (error) { - console.log('Warning: Failed to parse remaining JSON'); - console.log('Buffer:', buffer); - } - } -} - -interface Session { - start: number; - end: number; - profileId?: string; - deviceId?: string; - sessionId: string; - firstEvent?: IImportedEvent; - lastEvent?: IImportedEvent; - events: IImportedEvent[]; -} - -function generateSessionEvents(events: IImportedEvent[]): Session[] { - let sessionList: Session[] = []; - const lastSessionByDevice: Record = {}; - const lastSessionByProfile: Record = {}; - const thirtyMinutes = 30 * 60 * 1000; - - events.sort( - (a, b) => - new Date(a.created_at).getTime() - new Date(b.created_at).getTime(), - ); - - for (const event of events) { - const eventTime = new Date(event.created_at).getTime(); - let deviceSession = event.device_id - ? lastSessionByDevice[event.device_id] - : undefined; - let profileSession = event.profile_id - ? lastSessionByProfile[event.profile_id] - : undefined; - - if ( - event.device_id && - event.device_id !== event.profile_id && - (!deviceSession || eventTime > deviceSession.end + thirtyMinutes) - ) { - deviceSession = { - start: eventTime, - end: eventTime, - deviceId: event.device_id, - sessionId: randomUUID(), - firstEvent: event, - events: [event], - }; - lastSessionByDevice[event.device_id] = deviceSession; - sessionList.push(deviceSession); - } else if (deviceSession) { - deviceSession.end = eventTime; - deviceSession.lastEvent = event; - deviceSession.events.push(event); - } - - if ( - event.profile_id && - event.device_id !== event.profile_id && - (!profileSession || eventTime > profileSession.end + thirtyMinutes) - ) { - profileSession = { - start: eventTime, - end: eventTime, - profileId: event.profile_id, - sessionId: randomUUID(), - firstEvent: event, - events: [event], - }; - lastSessionByProfile[event.profile_id] = profileSession; - sessionList.push(profileSession); - } else if (profileSession) { - profileSession.end = eventTime; - profileSession.lastEvent = event; - profileSession.events.push(event); - } - - if ( - deviceSession && - profileSession && - deviceSession.sessionId !== profileSession.sessionId - ) { - const unifiedSession = { - ...deviceSession, - ...profileSession, - events: [...deviceSession.events, ...profileSession.events], - start: Math.min(deviceSession.start, profileSession.start), - end: Math.max(deviceSession.end, profileSession.end), - sessionId: deviceSession.sessionId, - }; - lastSessionByDevice[event.device_id] = unifiedSession; - lastSessionByProfile[event.profile_id] = unifiedSession; - sessionList = sessionList.filter( - (session) => - session.sessionId !== deviceSession?.sessionId && - session.sessionId !== profileSession?.sessionId, - ); - sessionList.push(unifiedSession); - } - } - - return sessionList; -} - -function createEventObject(event: IMixpanelEvent): IImportedEvent { - const getReferrer = (referrer: string | undefined) => { - if (!referrer) { - return ''; - } - - if (referrer === '$direct') { - return ''; - } - - if (isSameDomain(referrer, event.properties.$current_url)) { - return ''; - } - - return referrer; - }; - const url = parsePath(event.properties.$current_url); - return { - profile_id: event.properties.distinct_id - ? String(event.properties.distinct_id).replace(/^\$device:/, '') - : (event.properties.$device_id ?? ''), - name: event.event, - created_at: new Date(event.properties.time * 1000).toISOString(), - properties: { - ...stripMixpanelProperties(event.properties), - ...(event.properties.$current_url - ? { - __query: url.query, - __hash: url.hash, - } - : {}), - }, - country: event.properties.country_code ?? '', - region: event.properties.$region ?? '', - city: event.properties.$city ?? '', - os: event.properties.$os ?? '', - browser: event.properties.$browser ?? '', - browser_version: event.properties.$browser_version - ? String(event.properties.$browser_version) - : '', - referrer: getReferrer(event.properties.$initial_referrer), - referrer_type: event.properties.$search_engine ? 'search' : '', - referrer_name: event.properties.$search_engine ?? '', - device_id: event.properties.$device_id ?? '', - session_id: '', - project_id: '', - path: url.path, - origin: url.origin, - os_version: '', - model: '', - longitude: null, - latitude: null, - id: randomUUID(), - duration: 0, - device: event.properties.$current_url ? '' : 'server', - brand: '', - sdk_name: '', - sdk_version: '', - }; -} - -function isMixpanelEvent(event: any): event is IMixpanelEvent { - return ( - typeof event === 'object' && - event !== null && - typeof event?.event === 'string' && - typeof event?.properties === 'object' && - event?.properties !== null && - typeof event?.properties.time === 'number' - ); -} - -async function processFile(file: string): Promise { - const fileStream = fs.createReadStream(file); - const events: IImportedEvent[] = []; - for await (const event of parseJsonStream(fileStream)) { - if (Array.isArray(event)) { - for (const item of event) { - if (isMixpanelEvent(item)) { - events.push(createEventObject(item)); - } else { - console.log('Not a Mixpanel event', item); - } - } - } else { - if (isMixpanelEvent(event)) { - events.push(createEventObject(event)); - } else { - console.log('Not a Mixpanel event', event); - } - } - } - return events; -} - -function processEvents(events: IImportedEvent[]): IImportedEvent[] { - const sessions = generateSessionEvents(events); - const processedEvents = sessions.flatMap((session) => - [ - session.firstEvent && { - ...session.firstEvent, - id: randomUUID(), - created_at: new Date( - new Date(session.firstEvent.created_at).getTime() - 1000, - ).toISOString(), - session_id: session.sessionId, - name: 'session_start', - }, - ...uniqBy( - prop('id'), - session.events.map((event) => - assocPath(['session_id'], session.sessionId, event), - ), - ), - session.lastEvent && { - ...session.lastEvent, - id: randomUUID(), - created_at: new Date( - new Date(session.lastEvent.created_at).getTime() + 1000, - ).toISOString(), - session_id: session.sessionId, - name: 'session_end', - }, - ].filter((item): item is IImportedEvent => !!item), - ); - - return [ - ...processedEvents, - ...events.filter((event) => { - return !event.profile_id && !event.device_id; - }), - ]; -} - -async function sendBatchToAPI( - batch: IImportedEvent[], - { - apiUrl, - clientId, - clientSecret, - }: { - apiUrl: string; - clientId: string; - clientSecret: string; - }, -) { - async function request() { - const res = await fetch(`${apiUrl}/import/events`, { - method: 'POST', - headers: { - 'Content-Encoding': 'gzip', - 'Content-Type': 'application/json', - 'openpanel-client-id': clientId, - 'openpanel-client-secret': clientSecret, - }, - body: Buffer.from(zlib.gzipSync(JSON.stringify(batch))), - }); - if (!res.ok) { - throw new Error(`Failed to send batch: ${await res.text()}`); - } - await new Promise((resolve) => setTimeout(resolve, SLEEP_TIME)); - } - - try { - await request(); - } catch (e) { - console.log('Error sending batch, retrying...'); - await new Promise((resolve) => setTimeout(resolve, 1000)); - try { - await request(); - } catch (e) { - console.log('Error sending batch, skipping...'); - fs.writeFileSync( - path.join( - os.tmpdir(), - `openpanel/failed-import-batch-${batch[0]?.created_at ? new Date(batch[0]?.created_at).toISOString() : Date.now()}.json`, - ), - JSON.stringify(batch, null, 2), - ); - } - } -} - -async function processFiles({ - files, - apiUrl, - clientId, - clientSecret, -}: { - files: string[]; - apiUrl: string; - clientId: string; - clientSecret: string; -}) { - const progress = new Progress( - 'Processing (:current/:total) :file [:bar] :percent | :savedEvents saved events | :status', - { - total: files.length, - width: 20, - }, - ); - let savedEvents = 0; - let currentBatch: IImportedEvent[] = []; - let apiBatching = []; - - for (const file of files) { - progress.tick({ - file, - savedEvents, - status: 'reading file', - }); - const events = await processFile(file); - progress.render({ - file, - savedEvents, - status: 'processing events', - }); - const processedEvents = processEvents(events); - for (const event of processedEvents) { - currentBatch.push(event); - if (currentBatch.length >= BATCH_SIZE) { - apiBatching.push(currentBatch); - savedEvents += currentBatch.length; - progress.render({ file, savedEvents, status: 'saving events' }); - currentBatch = []; - } - - if (apiBatching.length >= MAX_CONCURRENT_REQUESTS) { - await Promise.all( - apiBatching.map((batch) => - sendBatchToAPI(batch, { - apiUrl, - clientId, - clientSecret, - }), - ), - ); - apiBatching = []; - } - } - } - - if (currentBatch.length > 0) { - await sendBatchToAPI(currentBatch, { - apiUrl, - clientId, - clientSecret, - }); - savedEvents += currentBatch.length; - progress.render({ file: 'Complete', savedEvents, status: 'Complete' }); - } -} - -export async function importFiles({ - files, - apiUrl, - clientId, - clientSecret, -}: { - files: string[]; - apiUrl: string; - clientId: string; - clientSecret: string; -}) { - if (files.length === 0) { - console.log('No files found'); - return; - } - - console.log(`Found ${files.length} files to process`); - - const startTime = Date.now(); - await processFiles({ - files, - apiUrl, - clientId, - clientSecret, - }); - const endTime = Date.now(); - - console.log( - `\nProcessing completed in ${(endTime - startTime) / 1000} seconds`, - ); -} diff --git a/packages/cli/src/importer/index.ts b/packages/cli/src/importer/index.ts deleted file mode 100644 index fcd1f196..00000000 --- a/packages/cli/src/importer/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import path from 'node:path'; -import arg from 'arg'; -import { glob } from 'glob'; - -import { importFiles } from './importer'; - -export default async function importer() { - const args = arg( - { - '--glob': String, - '--api-url': String, - '--client-id': String, - '--client-secret': String, - '--dry-run': Boolean, - '--from': Number, - '--to': Number, - }, - { - permissive: true, - }, - ); - - if (!args['--glob']) { - throw new Error('Missing --glob argument'); - } - - if (!args['--client-id']) { - throw new Error('Missing --client-id argument'); - } - - if (!args['--client-secret']) { - throw new Error('Missing --client-secret argument'); - } - - const cwd = process.cwd(); - - const fileMatcher = path.resolve(cwd, args['--glob']); - const allFiles = await glob([fileMatcher], { root: '/' }); - allFiles.sort((a, b) => a.localeCompare(b)); - - const files = allFiles.slice( - args['--from'] ?? 0, - args['--to'] ?? Number.MAX_SAFE_INTEGER, - ); - - if (args['--dry-run']) { - files.forEach((file, index) => { - console.log(`Would import (index: ${index}): ${file}`); - }); - return; - } - - return importFiles({ - files, - clientId: args['--client-id'], - clientSecret: args['--client-secret'], - apiUrl: args['--api-url'] ?? 'https://api.openpanel.dev', - }); -} diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json deleted file mode 100644 index fa4341f1..00000000 --- a/packages/cli/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "@openpanel/tsconfig/base.json", - "compilerOptions": { - "incremental": false, - "outDir": "dist" - }, - "exclude": ["dist"] -} diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts deleted file mode 100644 index 628edb0a..00000000 --- a/packages/cli/tsup.config.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { defineConfig } from 'tsup'; - -export default defineConfig({ - entry: ['src/cli.ts'], - format: ['cjs', 'esm'], - dts: true, - splitting: false, - sourcemap: false, - clean: true, - minify: true, -}); diff --git a/packages/common/package.json b/packages/common/package.json index a8c914f7..5eaab8d2 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -3,9 +3,14 @@ "version": "0.0.1", "type": "module", "main": "index.ts", + "exports": { + ".": "./index.ts", + "./server": "./server/index.ts" + }, "scripts": { "test": "vitest", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "gen:referrers": "jiti scripts/get-referrers.ts && biome format --write ./server/referrers/index.ts" }, "dependencies": { "@openpanel/constants": "workspace:*", diff --git a/packages/common/scripts/get-referrers.ts b/packages/common/scripts/get-referrers.ts new file mode 100644 index 00000000..27302eb7 --- /dev/null +++ b/packages/common/scripts/get-referrers.ts @@ -0,0 +1,96 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// extras +const extraReferrers = { + 'zoom.us': { type: 'social', name: 'Zoom' }, + 'apple.com': { type: 'tech', name: 'Apple' }, + 'adobe.com': { type: 'tech', name: 'Adobe' }, + 'figma.com': { type: 'tech', name: 'Figma' }, + 'wix.com': { type: 'commerce', name: 'Wix' }, + 'gmail.com': { type: 'email', name: 'Gmail' }, + 'notion.so': { type: 'tech', name: 'Notion' }, + 'ebay.com': { type: 'commerce', name: 'eBay' }, + 'github.com': { type: 'tech', name: 'GitHub' }, + 'gitlab.com': { type: 'tech', name: 'GitLab' }, + 'slack.com': { type: 'social', name: 'Slack' }, + 'etsy.com': { type: 'commerce', name: 'Etsy' }, + 'bsky.app': { type: 'social', name: 'Bluesky' }, + 'twitch.tv': { type: 'content', name: 'Twitch' }, + 'dropbox.com': { type: 'tech', name: 'Dropbox' }, + 'outlook.com': { type: 'email', name: 'Outlook' }, + 'medium.com': { type: 'content', name: 'Medium' }, + 'paypal.com': { type: 'commerce', name: 'PayPal' }, + 'discord.com': { type: 'social', name: 'Discord' }, + 'stripe.com': { type: 'commerce', name: 'Stripe' }, + 'spotify.com': { type: 'content', name: 'Spotify' }, + 'netflix.com': { type: 'content', name: 'Netflix' }, + 'whatsapp.com': { type: 'social', name: 'WhatsApp' }, + 'shopify.com': { type: 'commerce', name: 'Shopify' }, + 'microsoft.com': { type: 'tech', name: 'Microsoft' }, + 'alibaba.com': { type: 'commerce', name: 'Alibaba' }, + 'telegram.org': { type: 'social', name: 'Telegram' }, + 'substack.com': { type: 'content', name: 'Substack' }, + 'salesforce.com': { type: 'tech', name: 'Salesforce' }, + 'instagram.com': { type: 'social', name: 'Instagram' }, + 'wikipedia.org': { type: 'content', name: 'Wikipedia' }, + 'mastodon.social': { type: 'social', name: 'Mastodon' }, + 'office.com': { type: 'tech', name: 'Microsoft Office' }, + 'squarespace.com': { type: 'commerce', name: 'Squarespace' }, + 'stackoverflow.com': { type: 'tech', name: 'Stack Overflow' }, + 'teams.microsoft.com': { type: 'social', name: 'Microsoft Teams' }, +}; + +function transform(data: any) { + const obj: Record = {}; + for (const type in data) { + for (const name in data[type]) { + const domains = data[type][name].domains ?? []; + for (const domain of domains) { + obj[domain] = { + type, + name, + }; + } + } + } + + return obj; +} + +async function main() { + // Get document, or throw exception on error + try { + const data = await fetch( + 'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json', + ).then((res) => res.json()); + + fs.writeFileSync( + path.resolve(__dirname, '../../worker/src/referrers/index.ts'), + [ + '// This file is generated by the script get-referrers.ts', + '', + '// The data is fetch from snowplow-referer-parser https://github.com/snowplow-referer-parser/referer-parser', + `// The orginal referers.yml is based on Piwik's SearchEngines.php and Socials.php, copyright 2012 Matthieu Aubry and available under the GNU General Public License v3.`, + '', + `const referrers: Record = ${JSON.stringify( + { + ...transform(data), + ...extraReferrers, + }, + )} as const;`, + 'export default referrers;', + ].join('\n'), + 'utf-8', + ); + } catch (e) { + console.log(e); + } +} + +main(); diff --git a/packages/common/server/index.ts b/packages/common/server/index.ts index 3bb5bb37..cb540906 100644 --- a/packages/common/server/index.ts +++ b/packages/common/server/index.ts @@ -1,3 +1,5 @@ export * from './crypto'; export * from './profileId'; export * from './parser-user-agent'; +export * from './parse-referrer'; +export * from './id'; diff --git a/apps/worker/src/utils/parse-referrer.test.ts b/packages/common/server/parse-referrer.test.ts similarity index 93% rename from apps/worker/src/utils/parse-referrer.test.ts rename to packages/common/server/parse-referrer.test.ts index 66d17c54..009118b6 100644 --- a/apps/worker/src/utils/parse-referrer.test.ts +++ b/packages/common/server/parse-referrer.test.ts @@ -5,13 +5,13 @@ describe('parseReferrer', () => { it('should handle undefined or empty URLs', () => { expect(parseReferrer(undefined)).toEqual({ name: '', - type: 'unknown', + type: '', url: '', }); expect(parseReferrer('')).toEqual({ name: '', - type: 'unknown', + type: '', url: '', }); }); @@ -41,7 +41,7 @@ describe('parseReferrer', () => { it('should handle unknown referrers', () => { expect(parseReferrer('https://unknown-site.com')).toEqual({ name: '', - type: 'unknown', + type: '', url: 'https://unknown-site.com', }); }); @@ -49,7 +49,7 @@ describe('parseReferrer', () => { it('should handle invalid URLs', () => { expect(parseReferrer('not-a-url')).toEqual({ name: '', - type: 'unknown', + type: '', url: 'not-a-url', }); }); @@ -64,7 +64,7 @@ describe('getReferrerWithQuery', () => { it('should parse utm_source parameter', () => { expect(getReferrerWithQuery({ utm_source: 'google' })).toEqual({ name: 'Google', - type: 'unknown', + type: 'search', url: '', }); }); @@ -88,7 +88,7 @@ describe('getReferrerWithQuery', () => { it('should handle case-insensitive matching', () => { expect(getReferrerWithQuery({ utm_source: 'GoOgLe' })).toEqual({ name: 'Google', - type: 'unknown', + type: 'search', url: '', }); }); @@ -96,7 +96,7 @@ describe('getReferrerWithQuery', () => { it('should handle unknown sources', () => { expect(getReferrerWithQuery({ utm_source: 'unknown-source' })).toEqual({ name: 'unknown-source', - type: 'unknown', + type: '', url: '', }); }); @@ -110,7 +110,7 @@ describe('getReferrerWithQuery', () => { }), ).toEqual({ name: 'Google', - type: 'unknown', + type: 'search', url: '', }); }); diff --git a/apps/worker/src/utils/parse-referrer.ts b/packages/common/server/parse-referrer.ts similarity index 68% rename from apps/worker/src/utils/parse-referrer.ts rename to packages/common/server/parse-referrer.ts index 6e4d86af..3fe35c5e 100644 --- a/apps/worker/src/utils/parse-referrer.ts +++ b/packages/common/server/parse-referrer.ts @@ -1,6 +1,6 @@ -import { stripTrailingSlash } from '@openpanel/common'; +import { stripTrailingSlash } from '../src/string'; -import referrers from '../referrers'; +import referrers from './referrers'; function getHostname(url: string | undefined) { if (!url) { @@ -20,7 +20,7 @@ export function parseReferrer(url: string | undefined) { return { name: match?.name ?? '', - type: match?.type ?? 'unknown', + type: match?.type ?? '', url: stripTrailingSlash(url ?? ''), }; } @@ -32,16 +32,23 @@ export function getReferrerWithQuery( return null; } - const source = query.utm_source ?? query.ref ?? query.utm_referrer ?? ''; + const source = ( + query.utm_source ?? + query.ref ?? + query.utm_referrer ?? + '' + ).toLowerCase(); if (source === '') { return null; } const match = + referrers[source] || + referrers[`${source}.com`] || Object.values(referrers).find( - (referrer) => referrer.name.toLowerCase() === source.toLowerCase(), - ) || referrers[source]; + (referrer) => referrer.name.toLowerCase() === source, + ); if (match) { return { @@ -53,7 +60,7 @@ export function getReferrerWithQuery( return { name: source, - type: 'unknown', + type: '', url: '', }; } diff --git a/packages/common/server/parser-user-agent.ts b/packages/common/server/parser-user-agent.ts index b84b9768..9238de3b 100644 --- a/packages/common/server/parser-user-agent.ts +++ b/packages/common/server/parser-user-agent.ts @@ -68,6 +68,7 @@ const parse = (ua: string): UAParser.IResult => { return res; }; +export type UserAgentInfo = ReturnType; export function parseUserAgent( ua?: string | null, overrides?: Record, @@ -80,13 +81,35 @@ export function parseUserAgent( } return { - os: overrides?.__os || res.os.name, - osVersion: overrides?.__osVersion || res.os.version, - browser: overrides?.__browser || res.browser.name, - browserVersion: overrides?.__browserVersion || res.browser.version, - device: overrides?.__device || res.device.type || getDevice(ua), - brand: overrides?.__brand || res.device.vendor, - model: overrides?.__model || res.device.model, + os: + typeof overrides?.__os === 'string' && overrides?.__os + ? overrides?.__os + : res.os.name, + osVersion: + typeof overrides?.__osVersion === 'string' && overrides?.__osVersion + ? overrides?.__osVersion + : res.os.version, + browser: + typeof overrides?.__browser === 'string' && overrides?.__browser + ? overrides?.__browser + : res.browser.name, + browserVersion: + typeof overrides?.__browserVersion === 'string' && + overrides?.__browserVersion + ? overrides?.__browserVersion + : res.browser.version, + device: + typeof overrides?.__device === 'string' && overrides?.__device + ? overrides?.__device + : res.device.type || getDevice(ua), + brand: + typeof overrides?.__brand === 'string' && overrides?.__brand + ? overrides?.__brand + : res.device.vendor, + model: + typeof overrides?.__model === 'string' && overrides?.__model + ? overrides?.__model + : res.device.model, isServer: false, } as const; } diff --git a/apps/worker/src/referrers/index.ts b/packages/common/server/referrers/index.ts similarity index 100% rename from apps/worker/src/referrers/index.ts rename to packages/common/server/referrers/index.ts diff --git a/apps/worker/src/referrers/referrers.readme.md b/packages/common/server/referrers/referrers.readme.md similarity index 100% rename from apps/worker/src/referrers/referrers.readme.md rename to packages/common/server/referrers/referrers.readme.md diff --git a/packages/common/src/object.test.ts b/packages/common/src/object.test.ts new file mode 100644 index 00000000..d33dcb78 --- /dev/null +++ b/packages/common/src/object.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { toDots } from './object'; + +describe('toDots', () => { + it('should convert an object to a dot object', () => { + const obj = { + a: 1, + b: 2, + array: ['1', '2', '3'], + arrayWithObjects: [{ a: 1 }, { b: 2 }, { c: 3 }], + objectWithArrays: { a: [1, 2, 3] }, + null: null, + undefined: undefined, + empty: '', + jsonString: '{"a": 1, "b": 2}', + }; + expect(toDots(obj)).toEqual({ + a: '1', + b: '2', + 'array.0': '1', + 'array.1': '2', + 'array.2': '3', + 'arrayWithObjects.0.a': '1', + 'arrayWithObjects.1.b': '2', + 'arrayWithObjects.2.c': '3', + 'objectWithArrays.a.0': '1', + 'objectWithArrays.a.1': '2', + 'objectWithArrays.a.2': '3', + 'jsonString.a': '1', + 'jsonString.b': '2', + }); + }); + + it('should handle malformed JSON strings gracefully', () => { + const obj = { + validJson: '{"key":"value"}', + malformedJson: '{"key":"unterminated string', + startsWithBrace: '{not json at all', + startsWithBracket: '[also not json', + regularString: 'normal string', + }; + + expect(toDots(obj)).toEqual({ + 'validJson.key': 'value', + regularString: 'normal string', + }); + }); +}); diff --git a/packages/common/src/object.ts b/packages/common/src/object.ts index dd3a237e..ebcb0917 100644 --- a/packages/common/src/object.ts +++ b/packages/common/src/object.ts @@ -1,5 +1,18 @@ import { anyPass, assocPath, isEmpty, isNil, reject } from 'ramda'; +function isValidJsonString(value: string): boolean { + return ( + (value.startsWith('{') && value.endsWith('}')) || + (value.startsWith('[') && value.endsWith(']')) + ); +} +function isMalformedJsonString(value: string): boolean { + return ( + (value.startsWith('{') && !value.endsWith('}')) || + (value.startsWith('[') && !value.endsWith(']')) + ); +} + export function toDots( obj: Record, path = '', @@ -19,10 +32,28 @@ export function toDots( }; } - if (value === undefined || value === null) { + if (value === undefined || value === null || value === '') { return acc; } + if (typeof value === 'string' && isMalformedJsonString(value)) { + // Skip it + return acc; + } + + // Fix nested json strings - but catch parse errors for malformed JSON + if (typeof value === 'string' && isValidJsonString(value)) { + try { + return { + ...acc, + ...toDots(JSON.parse(value), `${path}${key}.`), + }; + } catch { + // Skip it + return acc; + } + } + const cleanedValue = typeof value === 'string' ? removeInvalidSurrogates(value).trim() diff --git a/packages/db/code-migrations/5-add-imports-table.sql b/packages/db/code-migrations/5-add-imports-table.sql new file mode 100644 index 00000000..b9144b41 --- /dev/null +++ b/packages/db/code-migrations/5-add-imports-table.sql @@ -0,0 +1,47 @@ +CREATE TABLE IF NOT EXISTS events_imports_replicated ON CLUSTER '{cluster}' ( + `id` UUID DEFAULT generateUUIDv4(), + `name` LowCardinality(String), + `sdk_name` LowCardinality(String), + `sdk_version` LowCardinality(String), + `device_id` String CODEC(ZSTD(3)), + `profile_id` String CODEC(ZSTD(3)), + `project_id` String CODEC(ZSTD(3)), + `session_id` String CODEC(LZ4), + `path` String CODEC(ZSTD(3)), + `origin` String CODEC(ZSTD(3)), + `referrer` String CODEC(ZSTD(3)), + `referrer_name` String CODEC(ZSTD(3)), + `referrer_type` LowCardinality(String), + `duration` UInt64 CODEC(Delta(4), LZ4), + `properties` Map(String, String) CODEC(ZSTD(3)), + `created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)), + `country` LowCardinality(FixedString(2)), + `city` String, + `region` LowCardinality(String), + `longitude` Nullable(Float32) CODEC(Gorilla, LZ4), + `latitude` Nullable(Float32) CODEC(Gorilla, LZ4), + `os` LowCardinality(String), + `os_version` LowCardinality(String), + `browser` LowCardinality(String), + `browser_version` LowCardinality(String), + `device` LowCardinality(String), + `brand` LowCardinality(String), + `model` LowCardinality(String), + `imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4), + `import_id` String CODEC(ZSTD(3)), + `import_status` LowCardinality(String) DEFAULT 'pending', + `imported_at_meta` DateTime DEFAULT now() +) +ENGINE = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/openpanel/v1/{table}', '{replica}') +PARTITION BY toYYYYMM(imported_at_meta) +ORDER BY (import_id, created_at) +SETTINGS index_granularity = 8192; + +--- + +CREATE TABLE IF NOT EXISTS events_imports ON CLUSTER '{cluster}' AS events_imports_replicated +ENGINE = Distributed('{cluster}', currentDatabase(), events_imports_replicated, cityHash64(import_id)); + +--- + +ALTER TABLE events_imports_replicated ON CLUSTER '{cluster}' MODIFY TTL imported_at_meta + INTERVAL 7 DAY; \ No newline at end of file diff --git a/packages/db/code-migrations/5-add-imports-table.ts b/packages/db/code-migrations/5-add-imports-table.ts new file mode 100644 index 00000000..e9062c74 --- /dev/null +++ b/packages/db/code-migrations/5-add-imports-table.ts @@ -0,0 +1,90 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { TABLE_NAMES } from '../src/clickhouse/client'; +import { + createTable, + modifyTTL, + runClickhouseMigrationCommands, +} from '../src/clickhouse/migration'; +import { getIsCluster } from './helpers'; + +export async function up() { + const isClustered = getIsCluster(); + + const sqls: string[] = [ + ...createTable({ + name: 'events_imports', + columns: [ + // Same columns as events table + '`id` UUID DEFAULT generateUUIDv4()', + '`name` LowCardinality(String)', + '`sdk_name` LowCardinality(String)', + '`sdk_version` LowCardinality(String)', + '`device_id` String CODEC(ZSTD(3))', + '`profile_id` String CODEC(ZSTD(3))', + '`project_id` String CODEC(ZSTD(3))', + '`session_id` String CODEC(LZ4)', + '`path` String CODEC(ZSTD(3))', + '`origin` String CODEC(ZSTD(3))', + '`referrer` String CODEC(ZSTD(3))', + '`referrer_name` String CODEC(ZSTD(3))', + '`referrer_type` LowCardinality(String)', + '`duration` UInt64 CODEC(Delta(4), LZ4)', + '`properties` Map(String, String) CODEC(ZSTD(3))', + '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))', + '`country` LowCardinality(FixedString(2))', + '`city` String', + '`region` LowCardinality(String)', + '`longitude` Nullable(Float32) CODEC(Gorilla, LZ4)', + '`latitude` Nullable(Float32) CODEC(Gorilla, LZ4)', + '`os` LowCardinality(String)', + '`os_version` LowCardinality(String)', + '`browser` LowCardinality(String)', + '`browser_version` LowCardinality(String)', + '`device` LowCardinality(String)', + '`brand` LowCardinality(String)', + '`model` LowCardinality(String)', + '`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4)', + + // Additional metadata columns for import tracking + '`import_id` String CODEC(ZSTD(3))', + "`import_status` LowCardinality(String) DEFAULT 'pending'", + '`imported_at_meta` DateTime DEFAULT now()', + ], + orderBy: ['import_id', 'created_at'], + partitionBy: 'toYYYYMM(imported_at_meta)', + settings: { + index_granularity: 8192, + }, + distributionHash: 'cityHash64(import_id)', + replicatedVersion: '1', + isClustered, + }), + ]; + + // Add TTL policy for auto-cleanup after 7 days + sqls.push( + modifyTTL({ + tableName: 'events_imports', + isClustered, + ttl: 'imported_at_meta + INTERVAL 7 DAY', + }), + ); + + fs.writeFileSync( + path.join(__filename.replace('.ts', '.sql')), + sqls + .map((sql) => + sql + .trim() + .replace(/;$/, '') + .replace(/\n{2,}/g, '\n') + .concat(';'), + ) + .join('\n\n---\n\n'), + ); + + if (!process.argv.includes('--dry')) { + await runClickhouseMigrationCommands(sqls); + } +} diff --git a/packages/db/index.ts b/packages/db/index.ts index b0aef4e3..5a725e8c 100644 --- a/packages/db/index.ts +++ b/packages/db/index.ts @@ -1,5 +1,6 @@ export * from './src/prisma-client'; export * from './src/clickhouse/client'; +export * from './src/clickhouse/csv'; export * from './src/sql-builder'; export * from './src/services/chart.service'; export * from './src/services/clients.service'; @@ -23,5 +24,6 @@ export * from './src/services/access.service'; export * from './src/buffers'; export * from './src/types'; export * from './src/clickhouse/query-builder'; +export * from './src/services/import.service'; export * from './src/services/overview.service'; export * from './src/session-context'; diff --git a/packages/db/package.json b/packages/db/package.json index 56826516..284b6d96 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -13,7 +13,7 @@ "with-env": "dotenv -e ../../.env -c --" }, "dependencies": { - "@clickhouse/client": "^1.2.0", + "@clickhouse/client": "^1.12.1", "@openpanel/common": "workspace:*", "@openpanel/constants": "workspace:*", "@openpanel/json": "workspace:*", diff --git a/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql b/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql new file mode 100644 index 00000000..46fe9773 --- /dev/null +++ b/packages/db/prisma/migrations/20251018205153_add_import_table/migration.sql @@ -0,0 +1,22 @@ +-- CreateTable +CREATE TABLE "public"."imports" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "projectId" TEXT NOT NULL, + "provider" TEXT NOT NULL, + "sourceType" TEXT NOT NULL, + "sourceLocation" TEXT NOT NULL, + "jobId" TEXT, + "status" TEXT NOT NULL, + "config" JSONB NOT NULL DEFAULT '{}', + "totalEvents" INTEGER NOT NULL DEFAULT 0, + "processedEvents" INTEGER NOT NULL DEFAULT 0, + "errorMessage" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TIMESTAMP(3), + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "imports_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "public"."imports" ADD CONSTRAINT "imports_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/migrations/20251018214030_fix/migration.sql b/packages/db/prisma/migrations/20251018214030_fix/migration.sql new file mode 100644 index 00000000..1b8aebd8 --- /dev/null +++ b/packages/db/prisma/migrations/20251018214030_fix/migration.sql @@ -0,0 +1,13 @@ +/* + Warnings: + + - You are about to drop the column `provider` on the `imports` table. All the data in the column will be lost. + - You are about to drop the column `sourceLocation` on the `imports` table. All the data in the column will be lost. + - You are about to drop the column `sourceType` on the `imports` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "public"."imports" DROP COLUMN "provider", +DROP COLUMN "sourceLocation", +DROP COLUMN "sourceType", +ALTER COLUMN "config" DROP DEFAULT; diff --git a/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql b/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql new file mode 100644 index 00000000..ece57d17 --- /dev/null +++ b/packages/db/prisma/migrations/20251022191315_add_status_message/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."imports" ADD COLUMN "statusMessage" TEXT; diff --git a/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql b/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql new file mode 100644 index 00000000..b579e0f4 --- /dev/null +++ b/packages/db/prisma/migrations/20251028103556_import_batch_and_status/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "public"."imports" ADD COLUMN "currentBatch" INTEGER NOT NULL DEFAULT 0, +ADD COLUMN "currentStep" TEXT; diff --git a/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql b/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql new file mode 100644 index 00000000..c05db2ae --- /dev/null +++ b/packages/db/prisma/migrations/20251028150123_fix_imports_table/migration.sql @@ -0,0 +1,14 @@ +/* + Warnings: + + - Changed the type of `status` on the `imports` table. No cast exists, the column would be dropped and recreated, which cannot be done if there is data, since the column is required. + - Made the column `currentStep` on table `imports` required. This step will fail if there are existing NULL values in that column. + +*/ +-- CreateEnum +CREATE TYPE "public"."ImportStatus" AS ENUM ('pending', 'processing', 'completed', 'failed'); + +-- AlterTable +ALTER TABLE "public"."imports" DROP COLUMN "status", +ADD COLUMN "status" "public"."ImportStatus" NOT NULL, +ALTER COLUMN "currentStep" SET NOT NULL; diff --git a/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql b/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql new file mode 100644 index 00000000..6afc5787 --- /dev/null +++ b/packages/db/prisma/migrations/20251028150655_fix_imports_again/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."imports" ALTER COLUMN "currentStep" DROP NOT NULL; diff --git a/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql b/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql new file mode 100644 index 00000000..3abc525e --- /dev/null +++ b/packages/db/prisma/migrations/20251028152531_imports_again/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "public"."imports" ALTER COLUMN "currentBatch" DROP NOT NULL, +ALTER COLUMN "currentBatch" DROP DEFAULT, +ALTER COLUMN "currentBatch" SET DATA TYPE TEXT; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index def5169e..e6aee8ab 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -194,6 +194,7 @@ model Project { notificationRules NotificationRule[] notifications Notification[] + imports Import[] // When deleteAt > now(), the project will be deleted deleteAt DateTime? @@ -467,3 +468,31 @@ model ResetPassword { @@map("reset_password") } + +enum ImportStatus { + pending + processing + completed + failed +} + +model Import { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + projectId String + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + jobId String? // BullMQ job ID + status ImportStatus + statusMessage String? // Human-readable current step like "Importing events (Feb 2025)", "Generating session IDs" + errorMessage String? + /// [IPrismaImportConfig] + config Json + totalEvents Int @default(0) + processedEvents Int @default(0) + currentStep String? + currentBatch String? // String date 2020-01-01 + createdAt DateTime @default(now()) + completedAt DateTime? + updatedAt DateTime @default(now()) @updatedAt + + @@map("imports") +} diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.ts index f87cc903..d1be225c 100644 --- a/packages/db/src/buffers/base-buffer.ts +++ b/packages/db/src/buffers/base-buffer.ts @@ -1,4 +1,4 @@ -import { generateSecureId } from '@openpanel/common/server/id'; +import { generateSecureId } from '@openpanel/common/server'; import { type ILogger, createLogger } from '@openpanel/logger'; import { getRedisCache, runEvery } from '@openpanel/redis'; diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/db/src/buffers/event-buffer.test.ts index 0df5b4ae..1babc51b 100644 --- a/packages/db/src/buffers/event-buffer.test.ts +++ b/packages/db/src/buffers/event-buffer.test.ts @@ -10,6 +10,18 @@ import { } from 'vitest'; import { ch } from '../clickhouse/client'; +const clickhouseSettings = { + async_insert: 1, + http_headers_progress_interval_ms: '50000', + input_format_parallel_parsing: 1, + max_execution_time: 300, + max_http_get_redirects: '0', + max_insert_block_size: '500000', + send_progress_in_http_headers: 1, + wait_end_of_query: 1, + wait_for_async_insert: 1, +}; + // Mock transformEvent to avoid circular dependency with buffers -> services -> buffers vi.mock('../services/event.service', () => ({ transformEvent: (event: any) => ({ @@ -127,6 +139,7 @@ describe('EventBuffer with real Redis', () => { duration: 1000, }, ], + clickhouse_settings: clickhouseSettings, }); const sessionKey = `event_buffer:session:${first.session_id}`; @@ -171,6 +184,7 @@ describe('EventBuffer with real Redis', () => { format: 'JSONEachRow', table: 'events', values: [first, end], + clickhouse_settings: clickhouseSettings, }); const sessionKey = `event_buffer:session:${first.session_id}`; const storedEvents = await redis.lrange(sessionKey, 0, -1); @@ -502,6 +516,7 @@ describe('EventBuffer with real Redis', () => { format: 'JSONEachRow', table: 'events', values: [end], + clickhouse_settings: clickhouseSettings, }); const sessionKey = `event_buffer:session:${s}`; @@ -552,6 +567,7 @@ describe('EventBuffer with real Redis', () => { format: 'JSONEachRow', table: 'events', values: [view1, view2, view3, end], + clickhouse_settings: clickhouseSettings, }); // Session should be completely empty and removed @@ -596,6 +612,7 @@ describe('EventBuffer with real Redis', () => { format: 'JSONEachRow', table: 'events', values: [{ ...view1, duration: 1000 }], + clickhouse_settings: clickhouseSettings, }); // Session should be REMOVED from ready_sessions (only 1 event left) @@ -620,6 +637,7 @@ describe('EventBuffer with real Redis', () => { format: 'JSONEachRow', table: 'events', values: [{ ...view2, duration: 1000 }], + clickhouse_settings: clickhouseSettings, }); // Session should be REMOVED again (only 1 event left) @@ -667,6 +685,7 @@ describe('EventBuffer with real Redis', () => { format: 'JSONEachRow', table: 'events', values: [view, end], + clickhouse_settings: clickhouseSettings, }); // NOW it should be removed from ready_sessions (because it's empty) diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts index b363cb44..be55866a 100644 --- a/packages/db/src/clickhouse/client.ts +++ b/packages/db/src/clickhouse/client.ts @@ -1,3 +1,4 @@ +import { Readable } from 'node:stream'; import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client'; import { ClickHouseLogLevel, createClient } from '@clickhouse/client'; import sqlstring from 'sqlstring'; @@ -23,13 +24,10 @@ type WarnLogParams = LogParams & { err?: Error }; class CustomLogger implements Logger { trace({ message, args }: LogParams) { - logger.debug(message, args); + logger.info(message, args); } debug({ message, args }: LogParams) { - if (message.includes('Query:') && args?.response_status === 200) { - return; - } - logger.debug(message, args); + logger.info(message, args); } info({ message, args }: LogParams) { logger.info(message, args); @@ -56,14 +54,15 @@ export const TABLE_NAMES = { event_property_values_mv: 'event_property_values_mv', cohort_events_mv: 'cohort_events_mv', sessions: 'sessions', + events_imports: 'events_imports', }; export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = { max_open_connections: 30, - request_timeout: 60000, + request_timeout: 300000, keep_alive: { enabled: true, - idle_socket_ttl: 8000, + idle_socket_ttl: 60000, }, compression: { request: true, @@ -87,7 +86,7 @@ const cleanQuery = (query?: string) => ? query.replace(/\n/g, '').replace(/\s+/g, ' ').trim() : undefined; -async function withRetry( +export async function withRetry( operation: () => Promise, maxRetries = 3, baseDelay = 500, @@ -132,7 +131,34 @@ export const ch = new Proxy(originalCh, { const value = Reflect.get(target, property, receiver); if (property === 'insert') { - return (...args: any[]) => withRetry(() => value.apply(target, args)); + return (...args: any[]) => + withRetry(() => { + args[0].clickhouse_settings = { + // Allow bigger HTTP payloads/time to stream rows + async_insert: 1, + wait_for_async_insert: 1, + // Increase insert timeouts and buffer sizes for large batches + max_execution_time: 300, + max_insert_block_size: '500000', + max_http_get_redirects: '0', + // Ensure JSONEachRow stays efficient + input_format_parallel_parsing: 1, + // Keep long-running inserts/queries from idling out at proxies by sending progress headers + send_progress_in_http_headers: 1, + http_headers_progress_interval_ms: '50000', + // Ensure server holds the connection until the query is finished + wait_end_of_query: 1, + ...args[0].clickhouse_settings, + }; + return value.apply(target, args); + }); + } + + if (property === 'command') { + return (...args: any[]) => + withRetry(() => { + return value.apply(target, args); + }); } return value; @@ -177,6 +203,34 @@ export async function chQueryWithMeta>( return response; } +export async function chInsertCSV(tableName: string, rows: string[]) { + try { + const now = performance.now(); + // Create a readable stream in binary mode for CSV (similar to EventBuffer) + const csvStream = Readable.from(rows.join('\n'), { + objectMode: false, + }); + + await ch.insert({ + table: tableName, + values: csvStream, + format: 'CSV', + clickhouse_settings: { + format_csv_allow_double_quotes: 1, + format_csv_allow_single_quotes: 0, + }, + }); + + logger.info('CSV Insert successful', { + elapsed: performance.now() - now, + rows: rows.length, + }); + } catch (error) { + logger.error('CSV Insert failed:', error); + throw error; + } +} + export async function chQuery>( query: string, clickhouseSettings?: ClickHouseSettings, diff --git a/packages/db/src/clickhouse/csv.ts b/packages/db/src/clickhouse/csv.ts new file mode 100644 index 00000000..21802f04 --- /dev/null +++ b/packages/db/src/clickhouse/csv.ts @@ -0,0 +1,53 @@ +// ClickHouse Map(String, String) format in CSV uses single quotes, not JSON double quotes +// Format: '{'key1':'value1','key2':'value2'}' +// Single quotes inside values must be escaped with backslash: \' +// We also need to escape newlines and control characters to prevent CSV parsing issues +const escapeMapValue = (str: string) => { + return str + .replace(/\\/g, '\\\\') // Escape backslashes first + .replace(/'/g, "\\'") // Escape single quotes + .replace(/\n/g, '\\n') // Escape newlines + .replace(/\r/g, '\\r') // Escape carriage returns + .replace(/\t/g, '\\t') // Escape tabs + .replace(/\0/g, '\\0'); // Escape null bytes +}; + +export const csvEscapeJson = ( + value: Record | null | undefined, +): string => { + if (value == null) return ''; + + // Normalize to strings if your column is Map(String,String) + const normalized: Record = Object.fromEntries( + Object.entries(value).map(([k, v]) => [ + String(k), + v == null ? '' : String(v), + ]), + ); + + // Empty object should return empty Map (without quotes, csvEscapeField will handle if needed) + if (Object.keys(normalized).length === 0) return '{}'; + + const pairs = Object.entries(normalized) + .map(([k, v]) => `'${escapeMapValue(k)}':'${escapeMapValue(v)}'`) + .join(','); + + // Return Map format without outer quotes - csvEscapeField will handle CSV escaping + // This allows csvEscapeField to properly wrap/escape the entire field if it contains newlines/quotes + return csvEscapeField(`{${pairs}}`); +}; + +// Escape a CSV field - wrap in double quotes if it contains commas, quotes, or newlines +// Double quotes inside must be doubled (""), per CSV standard +export const csvEscapeField = (value: string | number): string => { + const str = String(value); + + // If field contains commas, quotes, or newlines, it must be quoted + if (/[,"\n\r]/.test(str)) { + // Escape double quotes by doubling them + const escaped = str.replace(/"/g, '""'); + return `"${escaped}"`; + } + + return str; +}; diff --git a/packages/db/src/clickhouse/migration.ts b/packages/db/src/clickhouse/migration.ts index 2b65d561..57f27d34 100644 --- a/packages/db/src/clickhouse/migration.ts +++ b/packages/db/src/clickhouse/migration.ts @@ -115,6 +115,22 @@ ENGINE = Distributed('{cluster}', currentDatabase(), ${replicated(tableName)}, $ ]; } +export const modifyTTL = ({ + tableName, + isClustered, + ttl, +}: { + tableName: string; + isClustered: boolean; + ttl: string; +}) => { + if (isClustered) { + return `ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' MODIFY TTL ${ttl}`; + } + + return `ALTER TABLE ${tableName} MODIFY TTL ${ttl}`; +}; + /** * Generates ALTER TABLE statements for adding columns */ diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index a182b69b..6cd68a80 100644 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -141,6 +141,10 @@ export type IServiceCreateEventPayload = Omit< IServiceEvent, 'id' | 'importedAt' | 'profile' | 'meta' >; +export type IServiceImportedEventPayload = Omit< + IServiceEvent, + 'profile' | 'meta' +>; export interface IServiceEvent { id: string; diff --git a/packages/db/src/services/import.service.ts b/packages/db/src/services/import.service.ts new file mode 100644 index 00000000..6d0b8f55 --- /dev/null +++ b/packages/db/src/services/import.service.ts @@ -0,0 +1,784 @@ +import type { ILogger } from '@openpanel/logger'; +import sqlstring from 'sqlstring'; +import { + TABLE_NAMES, + ch, + chInsertCSV, + convertClickhouseDateToJs, + formatClickhouseDate, +} from '../clickhouse/client'; +import { csvEscapeField, csvEscapeJson } from '../clickhouse/csv'; +import { type Prisma, db } from '../prisma-client'; +import type { IClickhouseEvent } from './event.service'; + +export interface ImportStageResult { + importId: string; + totalEvents: number; + insertedEvents: number; +} + +export interface ImportProgress { + importId: string; + totalEvents: number; + insertedEvents: number; + status: 'pending' | 'processing' | 'processed' | 'failed'; +} + +/** + * Insert a batch of events into the imports staging table + */ +export async function insertImportBatch( + events: IClickhouseEvent[], + importId: string, +): Promise { + if (events.length === 0) { + return { importId, totalEvents: 0, insertedEvents: 0 }; + } + + // Important to have same order as events_imports table + // CSV format: properly quotes fields that need it + const csvRows = events.map((event) => { + // Properties need to be converted to JSON for Map(String, String) + // All fields must be CSV-escaped when joining with commas + const fields = [ + csvEscapeField(event.id || ''), + csvEscapeField(event.name), + csvEscapeField(event.sdk_name || ''), + csvEscapeField(event.sdk_version || ''), + csvEscapeField(event.device_id || ''), + csvEscapeField(event.profile_id || ''), + csvEscapeField(event.project_id || ''), + csvEscapeField(event.session_id || ''), + csvEscapeField(event.path), + csvEscapeField(event.origin || ''), + csvEscapeField(event.referrer || ''), + csvEscapeField(event.referrer_name || ''), + csvEscapeField(event.referrer_type || ''), + csvEscapeField(event.duration ?? 0), + csvEscapeJson(event.properties), + csvEscapeField(event.created_at), + csvEscapeField(event.country || ''), + csvEscapeField(event.city || ''), + csvEscapeField(event.region || ''), + csvEscapeField(event.longitude != null ? event.longitude : '\\N'), + csvEscapeField(event.latitude != null ? event.latitude : '\\N'), + csvEscapeField(event.os || ''), + csvEscapeField(event.os_version || ''), + csvEscapeField(event.browser || ''), + csvEscapeField(event.browser_version || ''), + csvEscapeField(event.device || ''), + csvEscapeField(event.brand || ''), + csvEscapeField(event.model || ''), + csvEscapeField('\\N'), // imported_at (Nullable) + csvEscapeField(importId), + csvEscapeField('pending'), // import_status + csvEscapeField(formatClickhouseDate(new Date())), // imported_at_meta (DateTime, not DateTime64, so no milliseconds) + ]; + return fields.join(','); + }); + + await chInsertCSV(TABLE_NAMES.events_imports, csvRows); + + return { + importId, + totalEvents: events.length, + insertedEvents: events.length, + }; +} + +/** + * Generate deterministic session IDs for events that don't have them + * Uses 30-minute time windows to create consistent session IDs across imports + * Only processes events where device != 'server' and session_id = '' + */ +export async function generateSessionIds( + importId: string, + from: string, +): Promise { + const rangeWhere = [ + 'import_id = {importId:String}', + "import_status = 'pending'", + "device != 'server'", + "session_id = ''", + from ? 'toDate(created_at) = {from:String}' : '', + ] + .filter(Boolean) + .join(' AND '); + + // Use SQL to generate deterministic session IDs based on device_id + 30-min time windows + // This ensures same events always get same session IDs regardless of import order + const updateQuery = ` + ALTER TABLE ${TABLE_NAMES.events_imports} + UPDATE session_id = lower(hex(MD5(concat( + device_id, + '-', + toString(toInt64(toUnixTimestamp(created_at) / 1800)) + )))) + WHERE ${rangeWhere} + `; + + await ch.command({ + query: updateQuery, + query_params: { importId, from }, + clickhouse_settings: { + wait_end_of_query: 1, + mutations_sync: '2', // Wait for mutation to complete on all replicas (critical!) + send_progress_in_http_headers: 1, + http_headers_progress_interval_ms: '50000', + }, + }); +} + +/** + * Reconstruct sessions using SQL-based logic + * This identifies session boundaries and creates session_start/session_end events + * session_start inherits all properties from the first event in the session + * session_end inherits all properties from the last event in the session and calculates duration + */ +export async function createSessionsStartEndEvents( + importId: string, + from: string, +): Promise { + // First, let's identify session boundaries and get first/last events for each session + const rangeWhere = [ + 'import_id = {importId:String}', + "import_status = 'pending'", + "session_id != ''", // Only process events that have session IDs + 'toDate(created_at) = {from:String}', + ] + .filter(Boolean) + .join(' AND '); + + // Use window functions to efficiently get first event (all fields) and last event (only changing fields) + // session_end only needs: properties, path, origin, created_at - the rest can be inherited from session_start + const sessionEventsQuery = ` + SELECT + device_id, + session_id, + project_id, + profile_id, + argMin((path, origin, referrer, referrer_name, referrer_type, properties, created_at, country, city, region, longitude, latitude, os, os_version, browser, browser_version, device, brand, model), created_at) AS first_event, + argMax((path, origin, properties, created_at), created_at) AS last_event_fields, + min(created_at) AS first_timestamp, + max(created_at) AS last_timestamp + FROM ${TABLE_NAMES.events_imports} + WHERE ${rangeWhere} + AND name NOT IN ('session_start', 'session_end') + GROUP BY session_id, device_id, project_id, profile_id + `; + + const sessionEventsResult = await ch.query({ + query: sessionEventsQuery, + query_params: { importId, from }, + format: 'JSONEachRow', + }); + + const sessionData = (await sessionEventsResult.json()) as Array<{ + device_id: string; + session_id: string; + project_id: string; + profile_id: string; + first_event: [ + // string, // id + // string, // name + string, // path + string, // origin + string, // referrer + string, // referrer_name + string, // referrer_type + // number, // duration + Record, // properties + string, // created_at + string, // country + string, // city + string, // region + number | null, // longitude + number | null, // latitude + string, // os + string, // os_version + string, // browser + string, // browser_version + string, // device + string, // brand + string, // model + // string, // sdk_name + // string, // sdk_version + // string, // imported_at + ]; + last_event_fields: [ + string, // path + string, // origin + Record, // properties + string, // created_at + ]; + first_timestamp: string; + last_timestamp: string; + }>; + + // Create session_start and session_end events + const sessionEvents: IClickhouseEvent[] = []; + + for (const session of sessionData) { + // Destructure first event tuple (all fields) + const [ + // firstId, + // firstName, + firstPath, + firstOrigin, + firstReferrer, + firstReferrerName, + firstReferrerType, + // firstDuration, + firstProperties, + firstCreatedAt, + firstCountry, + firstCity, + firstRegion, + firstLongitude, + firstLatitude, + firstOs, + firstOsVersion, + firstBrowser, + firstBrowserVersion, + firstDevice, + firstBrand, + firstModel, + // firstSdkName, + // firstSdkVersion, + // firstImportedAt, + ] = session.first_event; + + // Destructure last event fields (only the changing ones) + const [lastPath, lastOrigin, lastProperties, lastCreatedAt] = + session.last_event_fields; + + // Calculate duration in milliseconds + // Parse timestamps as Date objects to calculate duration + const firstTime = new Date(session.first_timestamp).getTime(); + const lastTime = new Date(session.last_timestamp).getTime(); + const durationMs = lastTime - firstTime; + + // Helper function to adjust timestamp by milliseconds without timezone conversion + const adjustTimestamp = (timestamp: string, offsetMs: number): string => { + // Parse the timestamp, adjust it, and format back to ClickHouse format + const date = convertClickhouseDateToJs(timestamp); + date.setTime(date.getTime() + offsetMs); + return formatClickhouseDate(date); + }; + + // Create session_start event - inherit everything from first event but change name + // Set created_at to 1 second before the first event + sessionEvents.push({ + id: crypto.randomUUID(), + name: 'session_start', + device_id: session.device_id, + profile_id: session.profile_id, + project_id: session.project_id, + session_id: session.session_id, + path: firstPath, + origin: firstOrigin, + referrer: firstReferrer, + referrer_name: firstReferrerName, + referrer_type: firstReferrerType, + duration: 0, // session_start always has 0 duration + properties: firstProperties as Record< + string, + string | number | boolean | null | undefined + >, + created_at: adjustTimestamp(session.first_timestamp, -1000), // 1 second before first event + country: firstCountry, + city: firstCity, + region: firstRegion, + longitude: firstLongitude, + latitude: firstLatitude, + os: firstOs, + os_version: firstOsVersion, + browser: firstBrowser, + browser_version: firstBrowserVersion, + device: firstDevice, + brand: firstBrand, + model: firstModel, + imported_at: new Date().toISOString(), + sdk_name: 'import-session-reconstruction', + sdk_version: '1.0.0', + }); + + // Create session_end event - inherit most from session_start, but use last event's path, origin, properties + // Set created_at to 1 second after the last event + sessionEvents.push({ + id: crypto.randomUUID(), + name: 'session_end', + device_id: session.device_id, + profile_id: session.profile_id, + project_id: session.project_id, + session_id: session.session_id, + path: lastPath, // From last event + origin: lastOrigin, // From last event + referrer: firstReferrer, // Same as session_start + referrer_name: firstReferrerName, // Same as session_start + referrer_type: firstReferrerType, // Same as session_start + duration: durationMs, + properties: lastProperties as Record< + string, + string | number | boolean | null | undefined + >, // From last event + created_at: adjustTimestamp(session.last_timestamp, 500), // 1 second after last event + country: firstCountry, // Same as session_start + city: firstCity, // Same as session_start + region: firstRegion, // Same as session_start + longitude: firstLongitude, // Same as session_start + latitude: firstLatitude, // Same as session_start + os: firstOs, // Same as session_start + os_version: firstOsVersion, // Same as session_start + browser: firstBrowser, // Same as session_start + browser_version: firstBrowserVersion, // Same as session_start + device: firstDevice, // Same as session_start + brand: firstBrand, // Same as session_start + model: firstModel, // Same as session_start + imported_at: new Date().toISOString(), + sdk_name: 'import-session-reconstruction', + sdk_version: '1.0.0', + }); + } + + // Insert session events into imports table + if (sessionEvents.length > 0) { + await insertImportBatch(sessionEvents, importId); + } +} + +/** + * Migrate all events from imports table to production events table + * This includes both original events and generated session events + */ +export async function moveImportsToProduction( + importId: string, + from: string, +): Promise { + // Build the WHERE clause for migration + // For session events (session_start/session_end), we don't filter by their created_at + // because they're created with adjusted timestamps (±1 second) that might fall outside + // the date range. Instead, we include them if their session_id has events in this range. + let whereClause = 'import_id = {importId:String}'; + + if (from) { + whereClause += ` AND ( + (toDate(created_at) = {from:String}) OR + ( + name IN ('session_start', 'session_end') AND + session_id IN ( + SELECT DISTINCT session_id + FROM ${TABLE_NAMES.events_imports} + WHERE import_id = {importId:String} + AND toDate(created_at) = {from:String} + AND name NOT IN ('session_start', 'session_end') + ) + ) + )`; + } + + const migrationQuery = ` + INSERT INTO ${TABLE_NAMES.events} ( + id, + name, + sdk_name, + sdk_version, + device_id, + profile_id, + project_id, + session_id, + path, + origin, + referrer, + referrer_name, + referrer_type, + duration, + properties, + created_at, + country, + city, + region, + longitude, + latitude, + os, + os_version, + browser, + browser_version, + device, + brand, + model, + imported_at + ) + SELECT + id, + name, + sdk_name, + sdk_version, + device_id, + profile_id, + project_id, + session_id, + path, + origin, + referrer, + referrer_name, + referrer_type, + duration, + properties, + created_at, + country, + city, + region, + longitude, + latitude, + os, + os_version, + browser, + browser_version, + device, + brand, + model, + imported_at + FROM ${TABLE_NAMES.events_imports} + WHERE ${whereClause} + ORDER BY created_at ASC + `; + + await ch.command({ + query: migrationQuery, + query_params: { importId, from }, + clickhouse_settings: { + wait_end_of_query: 1, + // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection. + send_progress_in_http_headers: 1, + // The interval of sending these progress headers. Here it is less than 60s, + http_headers_progress_interval_ms: '50000', + }, + }); +} + +export async function backfillSessionsToProduction( + importId: string, + from: string, +): Promise { + // After migrating events, populate the sessions table based on the migrated sessions + // We detect all session_ids involved in this import from the imports table, + // then aggregate over the production events to construct session rows. + const sessionsInsertQuery = ` + INSERT INTO ${TABLE_NAMES.sessions} ( + id, + project_id, + profile_id, + device_id, + created_at, + ended_at, + is_bounce, + entry_origin, + entry_path, + exit_origin, + exit_path, + screen_view_count, + revenue, + event_count, + duration, + country, + region, + city, + longitude, + latitude, + device, + brand, + model, + browser, + browser_version, + os, + os_version, + sign, + version, + properties, + utm_medium, + utm_source, + utm_campaign, + utm_content, + utm_term, + referrer, + referrer_name, + referrer_type + ) + SELECT + any(e.session_id) as id, + any(e.project_id) as project_id, + if(any(nullIf(e.profile_id, e.device_id)) IS NULL, any(e.profile_id), any(nullIf(e.profile_id, e.device_id))) as profile_id, + any(e.device_id) as device_id, + argMin(e.created_at, e.created_at) as created_at, + argMax(e.created_at, e.created_at) as ended_at, + if( + argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = '', + if(countIf(e.name = 'screen_view') > 1, true, false), + argMaxIf(e.properties['__bounce'], e.created_at, e.name = 'session_end') = 'true' + ) as is_bounce, + argMinIf(e.origin, e.created_at, e.name = 'session_start') as entry_origin, + argMinIf(e.path, e.created_at, e.name = 'session_start') as entry_path, + argMaxIf(e.origin, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_origin, + argMaxIf(e.path, e.created_at, e.name = 'session_end' OR e.name = 'screen_view') as exit_path, + countIf(e.name = 'screen_view') as screen_view_count, + 0 as revenue, + countIf(e.name != 'screen_view' AND e.name != 'session_start' AND e.name != 'session_end') as event_count, + sumIf(e.duration, name = 'session_end') AS duration, + argMinIf(e.country, e.created_at, e.name = 'session_start') as country, + argMinIf(e.region, e.created_at, e.name = 'session_start') as region, + argMinIf(e.city, e.created_at, e.name = 'session_start') as city, + argMinIf(e.longitude, e.created_at, e.name = 'session_start') as longitude, + argMinIf(e.latitude, e.created_at, e.name = 'session_start') as latitude, + argMinIf(e.device, e.created_at, e.name = 'session_start') as device, + argMinIf(e.brand, e.created_at, e.name = 'session_start') as brand, + argMinIf(e.model, e.created_at, e.name = 'session_start') as model, + argMinIf(e.browser, e.created_at, e.name = 'session_start') as browser, + argMinIf(e.browser_version, e.created_at, e.name = 'session_start') as browser_version, + argMinIf(e.os, e.created_at, e.name = 'session_start') as os, + argMinIf(e.os_version, e.created_at, e.name = 'session_start') as os_version, + 1 as sign, + 1 as version, + argMinIf(e.properties, e.created_at, e.name = 'session_start') as properties, + argMinIf(e.properties['__query.utm_medium'], e.created_at, e.name = 'session_start') as utm_medium, + argMinIf(e.properties['__query.utm_source'], e.created_at, e.name = 'session_start') as utm_source, + argMinIf(e.properties['__query.utm_campaign'], e.created_at, e.name = 'session_start') as utm_campaign, + argMinIf(e.properties['__query.utm_content'], e.created_at, e.name = 'session_start') as utm_content, + argMinIf(e.properties['__query.utm_term'], e.created_at, e.name = 'session_start') as utm_term, + argMinIf(e.referrer, e.created_at, e.name = 'session_start') as referrer, + argMinIf(e.referrer_name, e.created_at, e.name = 'session_start') as referrer_name, + argMinIf(e.referrer_type, e.created_at, e.name = 'session_start') as referrer_type + FROM ${TABLE_NAMES.events_imports} e + WHERE + e.import_id = ${sqlstring.escape(importId)} + AND toDate(e.created_at) = ${sqlstring.escape(from)} + AND e.session_id != '' + GROUP BY e.session_id + `; + + await ch.command({ + query: sessionsInsertQuery, + clickhouse_settings: { + wait_end_of_query: 1, + // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection. + send_progress_in_http_headers: 1, + // The interval of sending these progress headers. Here it is less than 60s, + http_headers_progress_interval_ms: '50000', + }, + }); +} + +/** + * Mark import as complete by updating status + */ +export async function markImportComplete(importId: string): Promise { + const updateQuery = ` + ALTER TABLE ${TABLE_NAMES.events_imports} + UPDATE import_status = 'processed' + WHERE import_id = {importId:String} + `; + + await ch.command({ + query: updateQuery, + query_params: { importId }, + clickhouse_settings: { + wait_end_of_query: 1, + mutations_sync: '2', // Wait for mutation to complete + // Ask ClickHouse to periodically send query execution progress in HTTP headers, creating some activity in the connection. + send_progress_in_http_headers: 1, + // The interval of sending these progress headers. Here it is less than 60s, + http_headers_progress_interval_ms: '50000', + }, + }); +} + +/** + * Get import progress and status + */ +export async function getImportProgress( + importId: string, +): Promise { + const progressQuery = ` + SELECT + import_id, + COUNT(*) as total_events, + COUNTIf(import_status = 'pending') as pending_events, + COUNTIf(import_status = 'processed') as processed_events, + any(import_status) as status + FROM ${TABLE_NAMES.events_imports} + WHERE import_id = {importId:String} + AND name NOT IN ('session_start', 'session_end') + GROUP BY import_id + `; + + const result = await ch.query({ + query: progressQuery, + query_params: { importId }, + format: 'JSONEachRow', + }); + + const data = (await result.json()) as Array<{ + import_id: string; + total_events: number; + pending_events: number; + processed_events: number; + status: string; + }>; + + if (data.length === 0) { + return { + importId, + totalEvents: 0, + insertedEvents: 0, + status: 'pending', + }; + } + + const row = data[0]; + if (!row) { + return { + importId, + totalEvents: 0, + insertedEvents: 0, + status: 'pending', + }; + } + + return { + importId, + totalEvents: row.total_events, + insertedEvents: row.processed_events, + status: row.status as 'pending' | 'processing' | 'processed' | 'failed', + }; +} + +/** + * Utility: get min/max created_at for an import + */ +export async function getImportDateBounds( + importId: string, + fromCreatedAt?: string, +): Promise<{ min: string | null; max: string | null }> { + const res = await ch.query({ + query: ` + SELECT min(created_at) AS min, max(created_at) AS max + FROM ${TABLE_NAMES.events_imports} + WHERE import_id = {importId:String} + ${fromCreatedAt ? 'AND created_at >= {fromCreatedAt:String}' : ''} + `, + query_params: { importId, fromCreatedAt }, + format: 'JSONEachRow', + }); + const rows = (await res.json()) as Array<{ + min: string | null; + max: string | null; + }>; + return rows.length > 0 + ? { + min: fromCreatedAt ?? rows[0]?.min ?? null, + max: rows[0]?.max ?? null, + } + : { min: null, max: null }; +} + +/** + * Unified method to update all import status information + * Combines step, batch, progress, and status message updates + */ +export type UpdateImportStatusOptions = + | { + step: 'loading'; + batch?: string; + totalEvents?: number; + processedEvents?: number; + } + | { + step: 'generating_session_ids'; + batch?: string; + } + | { + step: 'creating_sessions'; + batch?: string; + } + | { + step: 'moving'; + batch?: string; + } + | { + step: 'backfilling_sessions'; + batch?: string; + } + | { + step: 'completed'; + } + | { + step: 'failed'; + errorMessage?: string; + }; + +export type ImportSteps = UpdateImportStatusOptions['step']; + +export async function updateImportStatus( + jobLogger: ILogger, + job: { + updateProgress: (progress: Record) => void; + }, + importId: string, + options: UpdateImportStatusOptions, +): Promise { + const data: Prisma.ImportUpdateInput = {}; + switch (options.step) { + case 'loading': + data.status = 'processing'; + data.currentStep = 'loading'; + data.currentBatch = options.batch; + data.statusMessage = options.batch + ? `Importing events from ${options.batch}` + : 'Initializing...'; + data.totalEvents = options.totalEvents; + data.processedEvents = options.processedEvents; + break; + case 'generating_session_ids': + data.currentStep = 'generating_session_ids'; + data.currentBatch = options.batch; + data.statusMessage = options.batch + ? `Generating session IDs for ${options.batch}` + : 'Generating session IDs...'; + break; + case 'creating_sessions': + data.currentStep = 'creating_sessions'; + data.currentBatch = options.batch; + data.statusMessage = `Creating sessions for ${options.batch}`; + break; + case 'moving': + data.currentStep = 'moving'; + data.currentBatch = options.batch; + data.statusMessage = `Moving imports to production for ${options.batch}`; + break; + case 'backfilling_sessions': + data.currentStep = 'backfilling_sessions'; + data.currentBatch = options.batch; + data.statusMessage = `Aggregating sessions for ${options.batch}`; + break; + case 'completed': + data.status = 'completed'; + data.currentStep = 'completed'; + data.statusMessage = 'Import completed'; + data.completedAt = new Date(); + break; + case 'failed': + data.status = 'failed'; + data.statusMessage = 'Import failed'; + data.errorMessage = options.errorMessage; + break; + } + + jobLogger.info('Import status update', data); + + await job.updateProgress(data); + + await db.import.update({ + where: { id: importId }, + data, + }); +} diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.ts index 016e478e..aa9fe2fb 100644 --- a/packages/db/src/services/session.service.ts +++ b/packages/db/src/services/session.service.ts @@ -196,7 +196,7 @@ export async function getSessionList({ organization?.subscriptionPeriodEventsLimit && organization?.subscriptionPeriodEventsLimit > 1_000_000 ? 1 - : 7; + : 360; if (cursor) { const cAt = sqlstring.escape(cursor.createdAt); diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 8d6db586..7e525f25 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -1,4 +1,5 @@ import type { + IImportConfig, IIntegrationConfig, INotificationRuleConfig, IProjectFilters, @@ -12,6 +13,7 @@ import type { IClickhouseProfile } from './services/profile.service'; declare global { namespace PrismaJson { + type IPrismaImportConfig = IImportConfig; type IPrismaNotificationRuleConfig = INotificationRuleConfig; type IPrismaIntegrationConfig = IIntegrationConfig; type IPrismaNotificationPayload = INotificationPayload; diff --git a/packages/importer/package.json b/packages/importer/package.json new file mode 100644 index 00000000..ff91d681 --- /dev/null +++ b/packages/importer/package.json @@ -0,0 +1,35 @@ +{ + "name": "@openpanel/importer", + "version": "0.0.0", + "type": "module", + "main": "src/index.ts", + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "test": "vitest", + "test:run": "vitest run" + }, + "exports": { + ".": "./src/index.ts", + "./providers": "./src/providers/metadata.ts" + }, + "dependencies": { + "@openpanel/common": "workspace:*", + "@openpanel/db": "workspace:*", + "@openpanel/queue": "workspace:*", + "@openpanel/validation": "workspace:*", + "csv-parse": "^6.1.0", + "ramda": "^0.29.1", + "uuid": "^9.0.1", + "zod": "catalog:" + }, + "devDependencies": { + "@openpanel/logger": "workspace:*", + "@types/node": "^20.0.0", + "@types/ramda": "^0.31.1", + "@types/uuid": "^9.0.7", + "bullmq": "^5.8.7", + "typescript": "^5.0.0", + "vitest": "^1.0.0" + } +} diff --git a/packages/importer/src/base-provider.ts b/packages/importer/src/base-provider.ts new file mode 100644 index 00000000..e7c6189f --- /dev/null +++ b/packages/importer/src/base-provider.ts @@ -0,0 +1,121 @@ +import type { IClickhouseEvent } from '@openpanel/db'; +import type { BaseRawEvent, ErrorContext, ImportJobMetadata } from './types'; + +export abstract class BaseImportProvider< + TRawEvent extends BaseRawEvent = BaseRawEvent, +> { + abstract provider: string; + abstract version: string; + + /** + * Stream-read and parse source (file/API) → yields raw events + * This should be implemented as an async generator to handle large files efficiently + */ + abstract parseSource( + overrideFrom?: string, + ): AsyncGenerator; + + /** + * Convert provider format → IClickhouseEvent + */ + abstract transformEvent(rawEvent: TRawEvent): IClickhouseEvent; + + /** + * Validate raw event structure + */ + abstract validate(rawEvent: TRawEvent): boolean; + + /** + * Returns how many events will be imported + */ + abstract getTotalEventsCount(): Promise; + + /** + * Optional hook: Pre-process batch + */ + async beforeBatch?(events: TRawEvent[]): Promise { + return events; + } + + /** + * Optional hook: Get import metadata for tracking + */ + getImportMetadata?(): ImportJobMetadata; + + /** + * Optional hook: Custom error handling + */ + async onError?(error: Error, context?: ErrorContext): Promise { + // Default: re-throw + throw error; + } + + /** + * Get estimated total events (optional, for progress tracking) + */ + async getEstimatedTotal?(): Promise { + return 0; + } + + /** + * Indicates whether session IDs should be generated in SQL after import + * If true, the import job will generate deterministic session IDs based on + * device_id and timestamp using SQL window functions + * If false, assumes the provider already generates session IDs during streaming + */ + shouldGenerateSessionIds(): boolean { + return false; // Default: assume provider handles it + } + + /** + * Utility: Split a date range into chunks to avoid timeout issues with large imports + * Returns array of [from, to] date pairs in YYYY-MM-DD format + * + * @param from - Start date in YYYY-MM-DD format + * @param to - End date in YYYY-MM-DD format + * @param chunkSizeDays - Number of days per chunk (default: 1) + */ + public getDateChunks( + from: string, + to: string, + options?: { + chunkSizeDays?: number; + }, + ): Array<[string, string]> { + const chunks: Array<[string, string]> = []; + + const startDate = new Date(from); + const endDate = new Date(to); + const chunkSizeDays = options?.chunkSizeDays ?? 1; + + // Handle case where from and to are the same date + if (startDate.getTime() === endDate.getTime()) { + return [[from, to]]; + } + + const cursor = new Date(startDate); + + while (cursor <= endDate) { + const chunkStart = cursor.toISOString().split('T')[0]!; + + // Calculate chunk end: move forward by (chunkSizeDays - 1) to get the last day of the chunk + const chunkEndDate = new Date(cursor); + chunkEndDate.setDate(chunkEndDate.getDate() + (chunkSizeDays - 1)); + + // Don't go past the end date + const chunkEnd = + chunkEndDate > endDate + ? endDate.toISOString().split('T')[0]! + : chunkEndDate.toISOString().split('T')[0]!; + + chunks.push([chunkStart, chunkEnd]); + + // Move cursor to the next chunk start (after the current chunk) + cursor.setDate(cursor.getDate() + chunkSizeDays); + + if (cursor > endDate) break; + } + + return chunks; + } +} diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts new file mode 100644 index 00000000..52f4e80d --- /dev/null +++ b/packages/importer/src/index.ts @@ -0,0 +1,13 @@ +export { UmamiProvider } from './providers/umami'; +export { MixpanelProvider } from './providers/mixpanel'; +export type { + ImportConfig, + ImportProgress, + ImportResult, + BatchResult, + BaseRawEvent, + ErrorContext, + EventProperties, + ImportJobMetadata, + ImportStageResult, +} from './types'; diff --git a/packages/importer/src/providers/metadata.ts b/packages/importer/src/providers/metadata.ts new file mode 100644 index 00000000..639394b7 --- /dev/null +++ b/packages/importer/src/providers/metadata.ts @@ -0,0 +1,30 @@ +export type ImportProviderId = 'umami' | 'mixpanel'; +export type ImportProviderType = 'file' | 'api'; + +export interface ImportProviderMeta { + id: ImportProviderId; + name: string; + description: string; + logo: string; + backgroundColor: string; + types: ImportProviderType[]; +} + +export const IMPORT_PROVIDERS: ImportProviderMeta[] = [ + { + id: 'umami', + name: 'Umami', + description: 'Import your analytics data from Umami', + logo: 'https://cdn.brandfetch.io/id_3VEohOm/w/180/h/180/theme/dark/logo.png?c=1dxbfHSJFAPEGdCLU4o5B', + backgroundColor: '#fff', + types: ['file'], + }, + { + id: 'mixpanel', + name: 'Mixpanel', + description: 'Import your analytics data from Mixpanel API', + logo: 'https://cdn.brandfetch.io/idr_rhI2FS/theme/dark/idMJ8uODLv.svg?c=1dxbfHSJFAPEGdCLU4o5B', + backgroundColor: '#fff', + types: ['api'], + }, +]; diff --git a/packages/importer/src/providers/mixpanel.test.ts b/packages/importer/src/providers/mixpanel.test.ts new file mode 100644 index 00000000..c3f1d052 --- /dev/null +++ b/packages/importer/src/providers/mixpanel.test.ts @@ -0,0 +1,319 @@ +import { omit } from 'ramda'; +import { describe, expect, it } from 'vitest'; +import { MixpanelProvider } from './mixpanel'; + +describe('mixpanel', () => { + it('should chunk date range into day chunks', async () => { + const provider = new MixpanelProvider('pid', { + from: '2025-01-01', + to: '2025-01-04', + serviceAccount: 'sa', + serviceSecret: 'ss', + projectId: '123', + provider: 'mixpanel', + type: 'api', + mapScreenViewProperty: undefined, + }); + + const chunks = provider.getDateChunks('2025-01-01', '2025-01-04'); + expect(chunks).toEqual([ + ['2025-01-01', '2025-01-01'], + ['2025-01-02', '2025-01-02'], + ['2025-01-03', '2025-01-03'], + ['2025-01-04', '2025-01-04'], + ]); + }); + + it('should transform event', async () => { + const provider = new MixpanelProvider('pid', { + from: '2025-01-01', + to: '2025-01-02', + serviceAccount: 'sa', + serviceSecret: 'ss', + projectId: '123', + provider: 'mixpanel', + type: 'api', + mapScreenViewProperty: undefined, + }); + + const rawEvent = { + event: '$mp_web_page_view', + properties: { + time: 1746097970, + distinct_id: '$device:123', + $browser: 'Chrome', + $browser_version: 135, + $city: 'Mumbai', + $current_url: + 'https://domain.com/state/maharashtra?utm_source=google&utm_medium=cpc&utm_campaignid=890&utm_adgroupid=&utm_adid=&utm_term=&utm_device=m&utm_network=x&utm_location=123&gclid=oqneoqow&gad_sour', + $device: 'Android', + $device_id: '123', + $initial_referrer: 'https://referrer.com/', + $initial_referring_domain: 'referrer.com', + $insert_id: 'source_id', + $lib_version: '2.60.0', + $mp_api_endpoint: 'api-js.mixpanel.com', + $mp_api_timestamp_ms: 1746078175363, + $mp_autocapture: true, + $os: 'Android', + $referrer: 'https://google.com/', + $referring_domain: 'referrer.com', + $region: 'Maharashtra', + $screen_height: 854, + $screen_width: 384, + current_domain: 'domain.com', + current_page_title: + 'Landeed: Satbara Utara, 7/12 Extract, Property Card & Index 2', + current_url_path: '/state/maharashtra', + current_url_protocol: 'https:', + current_url_search: + '?utm_source=google&utm_medium=cpc&utm_campaignid=890&utm_adgroupid=&utm_adid=&utm_term=&utm_device=m&utm_network=x&utm_location=123&gclid=oqneoqow&gad_source=5&gclid=EAIaIQobChMI6MnvhciBjQMVlS-DAx', + gclid: 'oqneoqow', + mp_country_code: 'IN', + mp_lib: 'web', + mp_processing_time_ms: 1746078175546, + mp_sent_by_lib_version: '2.60.0', + utm_medium: 'cpc', + utm_source: 'google', + }, + }; + + const res = provider.transformEvent(rawEvent); + + expect(res).toMatchObject({ + id: expect.any(String), + name: 'screen_view', + device_id: '123', + profile_id: '123', + project_id: 'pid', + session_id: '', + properties: { + __source_insert_id: 'source_id', + __screen: '384x854', + __lib_version: '2.60.0', + '__query.utm_source': 'google', + '__query.utm_medium': 'cpc', + '__query.utm_campaignid': '890', + '__query.utm_device': 'm', + '__query.utm_network': 'x', + '__query.utm_location': '123', + '__query.gclid': 'oqneoqow', + __title: + 'Landeed: Satbara Utara, 7/12 Extract, Property Card & Index 2', + }, + created_at: '2025-05-01T11:12:50.000Z', + country: 'IN', + city: 'Mumbai', + region: 'Maharashtra', + longitude: null, + latitude: null, + os: 'Android', + os_version: undefined, + browser: 'Chrome', + browser_version: '', + device: 'mobile', + brand: '', + model: '', + duration: 0, + path: '/state/maharashtra', + origin: 'https://domain.com', + referrer: 'https://referrer.com', + referrer_name: 'Google', + referrer_type: 'search', + imported_at: expect.any(String), + sdk_name: 'mixpanel (web)', + sdk_version: '1.0.0', + }); + }); + + it('should parse stringified JSON in properties and flatten them', async () => { + const provider = new MixpanelProvider('pid', { + from: '2025-01-01', + to: '2025-01-02', + serviceAccount: 'sa', + serviceSecret: 'ss', + projectId: '123', + provider: 'mixpanel', + type: 'api', + mapScreenViewProperty: undefined, + }); + + const rawEvent = { + event: 'custom_event', + properties: { + time: 1746097970, + distinct_id: '$device:123', + $device_id: '123', + $user_id: 'user123', + mp_lib: 'web', + // Stringified JSON object - should be parsed and flattened + area: '{"displayText":"Malab, Nuh, Mewat","id":1189005}', + // Stringified JSON array - should be parsed and flattened + tags: '["tag1","tag2","tag3"]', + // Regular string - should remain as is + regularString: 'just a string', + // Number - should be converted to string + count: 42, + // Object - should be flattened + nested: { level1: { level2: 'value' } }, + }, + }; + + const res = provider.transformEvent(rawEvent); + + expect(res.properties).toMatchObject({ + // Parsed JSON object should be flattened with dot notation + 'area.displayText': 'Malab, Nuh, Mewat', + 'area.id': '1189005', + // Parsed JSON array should be flattened with numeric indices + 'tags.0': 'tag1', + 'tags.1': 'tag2', + 'tags.2': 'tag3', + // Regular values + regularString: 'just a string', + count: '42', + // Nested object flattened + 'nested.level1.level2': 'value', + }); + }); + + it('should handle react-native referrer', async () => { + const provider = new MixpanelProvider('pid', { + from: '2025-01-01', + to: '2025-01-02', + serviceAccount: 'sa', + serviceSecret: 'ss', + projectId: '123', + provider: 'mixpanel', + type: 'api', + mapScreenViewProperty: undefined, + }); + + const rawEvent = { + event: 'ec_search_error', + properties: { + time: 1759947367, + distinct_id: '3385916', + $browser: 'Mobile Safari', + $browser_version: null, + $city: 'Bengaluru', + $current_url: + 'https://web.landeed.com/karnataka/ec-encumbrance-certificate', + $device: 'iPhone', + $device_id: + '199b498af1036c-0e943279a1292e-5c0f4368-51bf4-199b498af1036c', + $initial_referrer: 'https://www.google.com/', + $initial_referring_domain: 'www.google.com', + $insert_id: 'bclkaepeqcfuzt4v', + $lib_version: '2.60.0', + $mp_api_endpoint: 'api-js.mixpanel.com', + $mp_api_timestamp_ms: 1759927570699, + $os: 'iOS', + $region: 'Karnataka', + $screen_height: 852, + $screen_width: 393, + $search_engine: 'google', + $user_id: '3385916', + binaryReadableVersion: 'NA', + binaryVersion: 'NA', + component: '/karnataka/ec-encumbrance-certificate', + errMsg: 'Request failed with status code 500', + errType: 'SERVER_ERROR', + isSilentSearch: false, + isTimeout: false, + jsVersion: '0.42.0', + language: 'english', + mp_country_code: 'IN', + mp_lib: 'web', + mp_processing_time_ms: 1759927592421, + mp_sent_by_lib_version: '2.60.0', + os: 'web', + osVersion: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1', + phoneBrand: 'NA', + phoneManufacturer: 'NA', + phoneModel: 'NA', + searchUuid: '68e65d08-fd81-4ded-37d3-2b08d2bc70c3', + serverVersion: 'web2.0', + state: 17, + stateStr: '17', + statusCode: 500, + type: 'result_event', + utm_medium: 'cpc', + utm_source: + 'google%26utm_medium=cpc%26utm_campaignid=21380769590%26utm_adgroupid=%26utm_adid=%26utm_term=%26utm_device=m%26utm_network=%26utm_location=9062055%26gclid=%26gad_campaignid=21374496705%26gbraid=0AAAAAoV7mTM9mWFripzQ2Od0xXAfrW6p3%26wbraid=CmAKCQjwi4PHBhCUA', + }, + }; + + const res = provider.transformEvent(rawEvent); + + expect(res.id.length).toBeGreaterThan(30); + expect(res.imported_at).toMatch( + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/, + ); + expect(omit(['id', 'imported_at'], res)).toEqual({ + brand: 'Apple', + browser: 'GSA', + browser_version: 'null', + city: 'Bengaluru', + country: 'IN', + created_at: '2025-10-08T18:16:07.000Z', + device: 'mobile', + device_id: '199b498af1036c-0e943279a1292e-5c0f4368-51bf4-199b498af1036c', + duration: 0, + latitude: null, + longitude: null, + model: 'iPhone', + name: 'ec_search_error', + origin: 'https://web.landeed.com', + os: 'iOS', + os_version: '18.7.0', + path: '/karnataka/ec-encumbrance-certificate', + profile_id: '3385916', + project_id: 'pid', + properties: { + __lib_version: '2.60.0', + '__query.gad_campaignid': '21374496705', + '__query.gbraid': '0AAAAAoV7mTM9mWFripzQ2Od0xXAfrW6p3', + '__query.utm_campaignid': '21380769590', + '__query.utm_device': 'm', + '__query.utm_location': '9062055', + '__query.utm_medium': 'cpc', + '__query.utm_source': 'google', + '__query.wbraid': 'CmAKCQjwi4PHBhCUA', + __screen: '393x852', + __source_insert_id: 'bclkaepeqcfuzt4v', + __userAgent: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1', + binaryReadableVersion: 'NA', + binaryVersion: 'NA', + component: '/karnataka/ec-encumbrance-certificate', + errMsg: 'Request failed with status code 500', + errType: 'SERVER_ERROR', + isSilentSearch: 'false', + isTimeout: 'false', + jsVersion: '0.42.0', + language: 'english', + os: 'web', + osVersion: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/388.0.811331708 Mobile/15E148 Safari/604.1', + phoneBrand: 'NA', + phoneManufacturer: 'NA', + phoneModel: 'NA', + searchUuid: '68e65d08-fd81-4ded-37d3-2b08d2bc70c3', + serverVersion: 'web2.0', + state: '17', + stateStr: '17', + statusCode: '500', + type: 'result_event', + }, + referrer: 'https://www.google.com', + referrer_name: 'Google', + referrer_type: 'search', + region: 'Karnataka', + sdk_name: 'mixpanel (web)', + sdk_version: '1.0.0', + session_id: '', + }); + }); +}); diff --git a/packages/importer/src/providers/mixpanel.ts b/packages/importer/src/providers/mixpanel.ts new file mode 100644 index 00000000..1d29bb33 --- /dev/null +++ b/packages/importer/src/providers/mixpanel.ts @@ -0,0 +1,452 @@ +import { randomUUID } from 'node:crypto'; +import { isSameDomain, parsePath, toDots } from '@openpanel/common'; +import { type UserAgentInfo, parseUserAgent } from '@openpanel/common/server'; +import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server'; +import type { IClickhouseEvent } from '@openpanel/db'; +import type { ILogger } from '@openpanel/logger'; +import type { IMixpanelImportConfig } from '@openpanel/validation'; +import { z } from 'zod'; +import { BaseImportProvider } from '../base-provider'; + +export const zMixpanelRawEvent = z.object({ + event: z.string(), + properties: z.record(z.unknown()), +}); + +export type MixpanelRawEvent = z.infer; + +export class MixpanelProvider extends BaseImportProvider { + provider = 'mixpanel'; + version = '1.0.0'; + + constructor( + private readonly projectId: string, + private readonly config: IMixpanelImportConfig, + private readonly logger?: ILogger, + ) { + super(); + } + + async getTotalEventsCount(): Promise { + // Mixpanel sucks and dont provide a good way to extract total event count within a period + // jql would work but not accurate and will be deprecated end of 2025 + return -1; + } + + /** + * Mixpanel doesn't provide session IDs, so we need to generate them in SQL + * after all events are imported to ensure deterministic results + */ + shouldGenerateSessionIds(): boolean { + return true; + } + + async *parseSource( + overrideFrom?: string, + ): AsyncGenerator { + yield* this.fetchEventsFromMixpanel(overrideFrom); + } + + private async *fetchEventsFromMixpanel( + overrideFrom?: string, + ): AsyncGenerator { + const { serviceAccount, serviceSecret, projectId, from, to } = this.config; + + // Split the date range into monthly chunks for reliability + // Uses base class utility to avoid timeout issues with large date ranges + const dateChunks = this.getDateChunks(overrideFrom ?? from, to); // 1 month per chunk + + for (const [chunkFrom, chunkTo] of dateChunks) { + yield* this.fetchEventsForDateRange( + serviceAccount, + serviceSecret, + projectId, + chunkFrom, + chunkTo, + ); + } + } + + private async *fetchEventsForDateRange( + serviceAccount: string, + serviceSecret: string, + projectId: string, + from: string, + to: string, + ): AsyncGenerator { + const url = 'https://data.mixpanel.com/api/2.0/export'; + + const params = new URLSearchParams({ + from_date: from, + to_date: to, + project_id: projectId, + }); + + this.logger?.info('Fetching events from Mixpanel', { + url: `${url}?${params}`, + from, + to, + projectId, + serviceAccount, + }); + + const response = await fetch(`${url}?${params}`, { + method: 'GET', + headers: { + Authorization: `Basic ${Buffer.from(`${serviceAccount}:${serviceSecret}`).toString('base64')}`, + Accept: 'application/json', + }, + }); + + if (!response.ok) { + throw new Error( + `Failed to fetch events from Mixpanel: ${response.status} ${response.statusText}`, + ); + } + + if (!response.body) { + throw new Error('No response body from Mixpanel API'); + } + + // Stream the response line by line + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + try { + while (true) { + const { done, value } = await reader.read(); + + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + // Process complete lines + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; // Keep the last incomplete line in buffer + + for (const line of lines) { + if (line.trim()) { + try { + const event = JSON.parse(line); + yield event; + } catch (error) { + console.warn('Failed to parse Mixpanel event:', line); + } + } + } + } + + // Process any remaining line in buffer + if (buffer.trim()) { + try { + const event = JSON.parse(buffer); + yield event; + } catch (error) { + console.warn('Failed to parse final Mixpanel event:', buffer); + } + } + } finally { + reader.releaseLock(); + } + } + + validate(rawEvent: MixpanelRawEvent): boolean { + const res = zMixpanelRawEvent.safeParse(rawEvent); + return res.success; + } + + transformEvent(_rawEvent: MixpanelRawEvent): IClickhouseEvent { + const projectId = this.projectId; + const rawEvent = zMixpanelRawEvent.parse(_rawEvent); + const props = rawEvent.properties as Record; + const deviceId = props.$device_id; + const profileId = String(props.$user_id || props.distinct_id).replace( + /^\$device:/, + '', + ); + + // Build full URL from current_url and current_url_search (web only) + const fullUrl = props.$current_url; + let path = ''; + let origin = ''; + let hash = ''; + let query: Record = {}; + + if (fullUrl) { + const parsed = parsePath(fullUrl); + path = parsed.path || ''; + origin = parsed.origin || ''; + hash = parsed.hash || ''; + query = parsed.query || {}; + } else if (this.config.mapScreenViewProperty) { + path = props[this.config.mapScreenViewProperty] || ''; + } + + // Extract referrer information (web only) + const referrerUrl = props.$initial_referrer || props.$referrer || ''; + const referrer = + referrerUrl && !isSameDomain(referrerUrl, fullUrl) + ? parseReferrer(referrerUrl) + : null; + + // Check for UTM referrer in query params (web only) + const utmReferrer = getReferrerWithQuery(query); + + // Extract location data + const country = props.$country || props.mp_country_code || ''; + const city = props.$city || ''; + const region = props.$region || ''; + + // For web events, use the standard user agent parsing + const userAgent = props.osVersion || ''; + const uaInfo = this.isWebEvent(props.mp_lib) + ? parseUserAgent(userAgent, props) + : this.parseServerDeviceInfo(props); + + // Map event name - $mp_web_page_view should be screen_view + let eventName = rawEvent.event; + if (eventName === '$mp_web_page_view') { + eventName = 'screen_view'; + } + + // Build properties object - strip Mixpanel-specific properties + const properties = this.stripMixpanelProperties(props, query); + + if (props.$insert_id) { + properties.__source_insert_id = String(props.$insert_id); + } + // Add useful properties + if (props.$screen_width && props.$screen_height) { + properties.__screen = `${props.$screen_width}x${props.$screen_height}`; + } + if (props.$screen_dpi) { + properties.__dpi = props.$screen_dpi; + } + if (props.$language) { + properties.__language = props.$language; + } + if (props.$timezone) { + properties.__timezone = props.$timezone; + } + if (props.$app_version) { + properties.__version = props.$app_version; + } + if (props.$app_build_number) { + properties.__buildNumber = props.$app_build_number; + } + if (props.$lib_version) { + properties.__lib_version = props.$lib_version; + } + + if (hash) { + properties.__hash = hash; + } + + if (Object.keys(query).length > 0) { + properties.__query = query; + } + + if (props.current_page_title) { + properties.__title = props.current_page_title; + } + + if (userAgent) { + properties.__userAgent = userAgent; + } + + // Always use UUID for id to match ClickHouse UUID column + const event = { + id: randomUUID(), + name: eventName, + device_id: deviceId, + profile_id: profileId, + project_id: projectId, + session_id: '', // Will be generated in SQL after import + properties: toDots(properties), // Flatten nested objects/arrays to Map(String, String) + created_at: new Date(props.time * 1000).toISOString(), + country, + city, + region, + longitude: null, + latitude: null, + os: uaInfo.os || props.$os, + os_version: uaInfo.osVersion || props.$osVersion, + browser: uaInfo.browser || props.$browser, + browser_version: + uaInfo.browserVersion || props.$browserVersion + ? String(props.$browser_version) + : '', + device: this.getDeviceType(props.mp_lib, uaInfo, props), + brand: uaInfo.brand || '', + model: uaInfo.model || '', + duration: 0, + path, + origin, + referrer: referrer?.url || '', + referrer_name: utmReferrer?.name || referrer?.name || '', + referrer_type: referrer?.type || utmReferrer?.type || '', + imported_at: new Date().toISOString(), + sdk_name: props.mp_lib + ? `${this.provider} (${props.mp_lib})` + : this.provider, + sdk_version: this.version, + }; + + // TODO: Remove this + // Temporary fix for a client + const isMightBeScreenView = this.getMightBeScreenView(rawEvent); + if (isMightBeScreenView && event.name === 'Loaded a Screen') { + event.name = 'screen_view'; + event.path = isMightBeScreenView; + } + + // TODO: Remove this + // This is a hack to get utm tags (not sure if this is just the testing project or all mixpanel projects) + if (props.utm_source && !properties.__query?.utm_source) { + const split = decodeURIComponent(props.utm_source).split('&'); + const query = Object.fromEntries(split.map((item) => item.split('='))); + for (const [key, value] of Object.entries(query)) { + if (key && value) { + event.properties[`__query.${key}`] = String(value); + } else if ( + value === undefined && + key && + props.utm_source.startsWith(key) + ) { + event.properties['__query.utm_source'] = String(key); + } + } + } + + return event; + } + + private getDeviceType( + mp_lib: string, + uaInfo: UserAgentInfo, + props: Record, + ) { + // Normalize lib/os/browser data + const lib = (mp_lib || '').toLowerCase(); + const os = String(props.$os || uaInfo.os || '').toLowerCase(); + const browser = String( + props.$browser || uaInfo.browser || '', + ).toLowerCase(); + + const isTabletOs = os === 'ipados' || os === 'ipad os' || os === 'ipad'; + + // Strong hint from SDK library + if (['android', 'iphone', 'react-native', 'swift', 'unity'].includes(lib)) { + return isTabletOs ? 'tablet' : 'mobile'; + } + + // Web or unknown SDKs: infer from OS/Browser + const isMobileSignal = + os === 'ios' || + os === 'android' || + browser.includes('mobile safari') || + browser.includes('chrome ios') || + browser.includes('android mobile') || + browser.includes('samsung internet') || + browser.includes('mobile'); + + if (isMobileSignal) { + return 'mobile'; + } + + const isTabletSignal = + isTabletOs || + browser.includes('tablet') || + // iPad often reports as Mac OS X with Mobile Safari + (browser.includes('mobile safari') && + (os === 'mac os x' || os === 'macos')); + + if (isTabletSignal) { + return 'tablet'; + } + + // Default to desktop + return this.isServerEvent(mp_lib) ? 'server' : 'desktop'; + } + + private isWebEvent(mp_lib: string) { + return [ + 'web', + 'android', + 'iphone', + 'swift', + 'unity', + 'react-native', + ].includes(mp_lib); + } + + private isServerEvent(mp_lib: string) { + return !this.isWebEvent(mp_lib); + } + + private getMightBeScreenView(rawEvent: MixpanelRawEvent) { + const props = rawEvent.properties as Record; + return Object.keys(props).find((key) => key.match(/^[A-Z1-9_]+$/)); + } + + private parseServerDeviceInfo(props: Record): UserAgentInfo { + // For mobile events, extract device information from Mixpanel properties + const os = props.$os || props.os || ''; + const osVersion = props.$os_version || props.osVersion || ''; + const brand = props.$brand || props.phoneBrand || ''; + const model = props.$model || props.phoneModel || ''; + const device = os.toLowerCase(); + + return { + isServer: true, + os: os, + osVersion: osVersion, + browser: '', + browserVersion: '', + device: device, + brand: brand, + model: model, + }; + } + + private stripMixpanelProperties( + properties: Record, + searchParams: Record, + ): Record { + const strip = [ + 'time', + 'distinct_id', + 'current_page_title', + 'current_url_path', + 'current_url_protocol', + 'current_url_search', + 'current_domain', + ...Object.keys(searchParams), + ]; + const filtered = Object.fromEntries( + Object.entries(properties).filter( + ([key]) => !key.match(/^(\$|mp_|utm_)/) && !strip.includes(key), + ), + ); + + // Parse JSON strings back to objects/arrays so toDots() can flatten them + const parsed: Record = {}; + for (const [key, value] of Object.entries(filtered)) { + if ( + typeof value === 'string' && + (value.startsWith('{') || value.startsWith('[')) + ) { + try { + parsed[key] = JSON.parse(value); + } catch { + parsed[key] = value; // Keep as string if parsing fails + } + } else { + parsed[key] = value; + } + } + + return parsed; + } +} diff --git a/packages/importer/src/providers/umami.ts b/packages/importer/src/providers/umami.ts new file mode 100644 index 00000000..f232e1b8 --- /dev/null +++ b/packages/importer/src/providers/umami.ts @@ -0,0 +1,382 @@ +import { randomUUID } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { createBrotliDecompress, createGunzip } from 'node:zlib'; +import { isSameDomain, parsePath } from '@openpanel/common'; +import { generateDeviceId } from '@openpanel/common/server'; +import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server'; +import type { IClickhouseEvent } from '@openpanel/db'; +import type { ILogger } from '@openpanel/logger'; +import type { IUmamiImportConfig } from '@openpanel/validation'; +import { parse } from 'csv-parse'; +import { assocPath } from 'ramda'; +import { z } from 'zod'; +import { BaseImportProvider } from '../base-provider'; + +export const zUmamiRawEvent = z.object({ + // Required fields + event_type: z.coerce.number(), + event_name: z.string(), + created_at: z.coerce.date(), + event_id: z.string().min(1), + session_id: z.string().min(1), + website_id: z.string().min(1), + + // Optional fields that might be empty + visit_id: z.string().optional(), + distinct_id: z.string().optional(), + url_path: z.string().optional(), + hostname: z.string().optional(), + referrer_domain: z.string().optional(), + referrer_path: z.string().optional(), + referrer_query: z.string().optional(), + referrer_name: z.string().optional(), + referrer_type: z.string().optional(), + country: z.string().optional(), + city: z.string().optional(), + region: z.string().optional(), + browser: z.string().optional(), + os: z.string().optional(), + device: z.string().optional(), + screen: z.string().optional(), + language: z.string().optional(), + utm_source: z.string().optional(), + utm_medium: z.string().optional(), + utm_campaign: z.string().optional(), + utm_content: z.string().optional(), + utm_term: z.string().optional(), + page_title: z.string().optional(), + gclid: z.string().optional(), + fbclid: z.string().optional(), + msclkid: z.string().optional(), + ttclid: z.string().optional(), + li_fat_id: z.string().optional(), + twclid: z.string().optional(), + url_query: z.string().optional(), +}); +export type UmamiRawEvent = z.infer; + +export class UmamiProvider extends BaseImportProvider { + provider = 'umami'; + version = '1.0.0'; + + constructor( + private readonly projectId: string, + private readonly config: IUmamiImportConfig, + private readonly logger?: ILogger, + ) { + super(); + } + + async getTotalEventsCount(): Promise { + return -1; + } + + async *parseSource(): AsyncGenerator { + yield* this.parseRemoteFile(this.config.fileUrl); + } + + private async *parseRemoteFile( + url: string, + opts: { + signal?: AbortSignal; + maxBytes?: number; + maxRows?: number; + } = {}, + ): AsyncGenerator { + const { signal, maxBytes, maxRows } = opts; + const controller = new AbortController(); + + // Link to caller's signal for cancellation + if (signal) { + signal.addEventListener('abort', () => controller.abort(), { + once: true, + }); + } + + const res = await fetch(url, { signal: controller.signal }); + if (!res.ok || !res.body) { + throw new Error( + `Failed to fetch remote file: ${res.status} ${res.statusText}`, + ); + } + + const contentType = res.headers.get('content-type') || ''; + const contentEnc = res.headers.get('content-encoding') || ''; + const contentLen = Number(res.headers.get('content-length') ?? 0); + + if ( + contentType && + !/text\/csv|text\/plain|application\/gzip|application\/octet-stream/i.test( + contentType, + ) + ) { + console.warn(`Warning: Content-Type is ${contentType}, expected CSV-ish`); + } + + if (maxBytes && contentLen && contentLen > maxBytes) { + throw new Error( + `Remote file exceeds size limit (${contentLen} > ${maxBytes})`, + ); + } + + const looksGzip = + /\.gz($|\?)/i.test(url) || + /gzip/i.test(contentEnc) || + /application\/gzip/i.test(contentType); + const looksBr = /br/i.test(contentEnc) || /\.br($|\?)/i.test(url); + + // WHATWG -> Node stream + const body = Readable.fromWeb(res.body as any); + + // Optional size guard during stream + let seenBytes = 0; + if (maxBytes) { + body.on('data', (chunk: Buffer) => { + seenBytes += chunk.length; + if (seenBytes > maxBytes) { + controller.abort(); + body.destroy( + new Error( + `Stream exceeded size limit (${seenBytes} > ${maxBytes})`, + ), + ); + } + }); + } + + // Build decode chain (gzip/brotli -> CSV parser) + const decompress = looksGzip + ? createGunzip() + : looksBr + ? createBrotliDecompress() + : null; + + const parser = parse({ + columns: true, // objects per row + bom: true, // handle UTF-8 BOM + relax_column_count: true, + skip_empty_lines: true, + }); + + // Wire the pipeline for proper backpressure & error propagation + (async () => { + try { + if (decompress) { + await pipeline(body, decompress, parser, { + signal: controller.signal, + }); + } else { + await pipeline(body, parser, { signal: controller.signal }); + } + } catch (e) { + parser.destroy(e as Error); + } + })().catch(() => { + /* handled by iterator */ + }); + + let rows = 0; + try { + for await (const record of parser) { + rows++; + if (maxRows && rows > maxRows) { + controller.abort(); + throw new Error(`Row limit exceeded (${rows} > ${maxRows})`); + } + yield record as UmamiRawEvent; + } + } catch (err) { + throw new Error( + `Failed to parse remote file from ${url}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } finally { + controller.abort(); // ensure fetch stream is torn down + } + } + + validate(rawEvent: UmamiRawEvent): boolean { + const res = zUmamiRawEvent.safeParse(rawEvent); + return res.success; + } + + transformEvent(_rawEvent: UmamiRawEvent): IClickhouseEvent { + const projectId = + this.config.projectMapper.find( + (mapper) => mapper.from === _rawEvent.website_id, + )?.to || this.projectId; + + const rawEvent = zUmamiRawEvent.parse(_rawEvent); + // Extract device/profile ID - use visit_id as device_id, session_id for session tracking + const deviceId = + rawEvent.visit_id || + generateDeviceId({ + ip: rawEvent.visit_id!, + ua: rawEvent.visit_id!, + origin: projectId, + salt: 'xxx', + }); + const profileId = rawEvent.distinct_id || deviceId; + + // Parse URL if available - use same logic as real-time events + const url = rawEvent.url_path + ? `https://${[rawEvent.hostname, rawEvent.url_path, rawEvent.url_query] + .filter(Boolean) + .join('')}` + : ''; + const { path, hash, query, origin } = parsePath(url); + // Extract referrer information - use same logic as real-time events + const referrerUrl = rawEvent.referrer_domain + ? `https://${rawEvent.referrer_domain}${rawEvent.referrer_path || ''}` + : ''; + + // Check if referrer is from same domain (like real-time events do) + const referrer = isSameDomain(referrerUrl, url) + ? null + : parseReferrer(referrerUrl); + + // Check for UTM referrer in query params (like real-time events do) + const utmReferrer = getReferrerWithQuery(query); + + // Extract location data + const country = rawEvent.country || ''; + const city = rawEvent.city || ''; + const region = rawEvent.region || ''; + + // Extract browser/device info + const browser = rawEvent.browser || ''; + const browserVersion = ''; // Not available in Umami CSV + const os = rawEvent.os || ''; + const osVersion = ''; // Not available in Umami CSV + const device = rawEvent.device || ''; + const brand = ''; // Not available in Umami CSV + const model = ''; // Not available in Umami CSV + + let properties: Record = {}; + + if (query) { + properties.__query = query; + } + + // Add useful properties from Umami data + if (rawEvent.page_title) properties.__title = rawEvent.page_title; + if (rawEvent.screen) properties.__screen = rawEvent.screen; + if (rawEvent.language) properties.__language = rawEvent.language; + if (rawEvent.utm_source) + properties = assocPath( + ['__query', 'utm_source'], + rawEvent.utm_source, + properties, + ); + if (rawEvent.utm_medium) + properties = assocPath( + ['__query', 'utm_medium'], + rawEvent.utm_medium, + properties, + ); + if (rawEvent.utm_campaign) + properties = assocPath( + ['__query', 'utm_campaign'], + rawEvent.utm_campaign, + properties, + ); + if (rawEvent.utm_content) + properties = assocPath( + ['__query', 'utm_content'], + rawEvent.utm_content, + properties, + ); + if (rawEvent.utm_term) + properties = assocPath( + ['__query', 'utm_term'], + rawEvent.utm_term, + properties, + ); + + return { + id: rawEvent.event_id || randomUUID(), + name: rawEvent.event_type === 1 ? 'screen_view' : rawEvent.event_name, + device_id: deviceId, + profile_id: profileId, + project_id: projectId, + session_id: rawEvent.session_id || '', + properties, + created_at: rawEvent.created_at.toISOString(), + country, + city, + region: this.mapRegion(region), + longitude: null, + latitude: null, + os, + os_version: osVersion, + browser: this.mapBrowser(browser), + browser_version: browserVersion, + device: this.mapDevice(device), + brand, + model, + duration: 0, + path, + origin, + referrer: utmReferrer?.url || referrer?.url || '', + referrer_name: utmReferrer?.name || referrer?.name || '', + referrer_type: utmReferrer?.type || referrer?.type || '', + imported_at: new Date().toISOString(), + sdk_name: this.provider, + sdk_version: this.version, + }; + } + + mapRegion(region: string): string { + return region.replace(/^[A-Z]{2}\-/, ''); + } + + mapDevice(device: string): string { + const mapping: Record = { + desktop: 'desktop', + laptop: 'desktop', + mobile: 'mobile', + tablet: 'tablet', + smarttv: 'smarttv', + Unknown: 'desktop', + }; + + return mapping[device] || 'desktop'; + } + + mapBrowser(browser: string): string { + const mapping: Record = { + android: 'Android', + aol: 'AOL', + bb10: 'BlackBerry 10', + beaker: 'Beaker', + chrome: 'Chrome', + 'chromium-webview': 'Chrome (webview)', + crios: 'Chrome (iOS)', + curl: 'Curl', + edge: 'Edge', + 'edge-chromium': 'Edge (Chromium)', + 'edge-ios': 'Edge (iOS)', + facebook: 'Facebook', + firefox: 'Firefox', + fxios: 'Firefox (iOS)', + ie: 'IE', + instagram: 'Instagram', + ios: 'iOS', + 'ios-webview': 'iOS (webview)', + kakaotalk: 'KakaoTalk', + miui: 'MIUI', + opera: 'Opera', + 'opera-mini': 'Opera Mini', + phantomjs: 'PhantomJS', + safari: 'Safari', + samsung: 'Samsung', + searchbot: 'Searchbot', + silk: 'Silk', + yandexbrowser: 'Yandex', + }; + + return mapping[browser] || browser || 'Unknown'; + } +} diff --git a/packages/importer/src/types.ts b/packages/importer/src/types.ts new file mode 100644 index 00000000..ae56bf29 --- /dev/null +++ b/packages/importer/src/types.ts @@ -0,0 +1,80 @@ +import type { + IImportedEvent, + IServiceCreateEventPayload, + IServiceImportedEventPayload, +} from '@openpanel/db'; + +export interface ImportConfig { + projectId: string; + provider: string; + sourceType: 'file' | 'api'; + sourceLocation: string; +} + +export interface SessionInfo { + id: string; + lastTimestamp: number; + lastEvent: IServiceImportedEventPayload; +} + +export interface ImportProgress { + totalEvents: number; + processedEvents: number; + currentBatch: number; + totalBatches: number; +} + +export interface ImportResult { + success: boolean; + totalEvents: number; + processedEvents: number; + error?: string; +} + +export interface BatchResult { + events: IServiceImportedEventPayload[]; + sessionEvents: IServiceImportedEventPayload[]; +} + +// Generic types for raw events from different providers +export interface BaseRawEvent { + [key: string]: unknown; +} + +// Error context for better error handling +export interface ErrorContext { + batchNumber?: number; + batchSize?: number; + eventIndex?: number; + rawEvent?: BaseRawEvent; + provider?: string; +} + +// Properties type for events - more specific than Record +export interface EventProperties { + [key: string]: + | string + | number + | boolean + | null + | undefined + | Record; + __query?: Record; + __title?: string; + __screen?: string; + __language?: string; +} + +// Import job metadata for tracking import progress +export interface ImportJobMetadata { + importId: string; + importStatus: 'pending' | 'processing' | 'processed' | 'failed'; + importedAt: Date; +} + +// Result of import staging operations +export interface ImportStageResult { + importId: string; + totalEvents: number; + insertedEvents: number; +} diff --git a/packages/importer/tsconfig.json b/packages/importer/tsconfig.json new file mode 100644 index 00000000..1d1748e9 --- /dev/null +++ b/packages/importer/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tooling/typescript/base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/importer/vitest.config.ts b/packages/importer/vitest.config.ts new file mode 100644 index 00000000..f87a2039 --- /dev/null +++ b/packages/importer/vitest.config.ts @@ -0,0 +1,3 @@ +import { getSharedVitestConfig } from '../../vitest.shared'; + +export default getSharedVitestConfig({ __dirname }); diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts index 7511cd09..9d07372a 100644 --- a/packages/queue/src/queues.ts +++ b/packages/queue/src/queues.ts @@ -110,7 +110,6 @@ export const eventsGroupQueue = new GroupQueue< >({ logger: queueLogger, namespace: 'group_events', - // @ts-expect-error - TODO: Fix this in groupmq redis: getRedisGroupQueue(), orderingMethod: 'in-memory', orderingWindowMs, @@ -166,6 +165,21 @@ export const notificationQueue = new Queue( }, ); +export type ImportQueuePayload = { + type: 'import'; + payload: { + importId: string; + }; +}; + +export const importQueue = new Queue('import', { + connection: getRedisQueue(), + defaultJobOptions: { + removeOnComplete: 10, + removeOnFail: 50, + }, +}); + export function addTrialEndingSoonJob(organizationId: string, delay: number) { return miscQueue.add( 'misc', diff --git a/packages/redis/package.json b/packages/redis/package.json index f958f1d7..742886ab 100644 --- a/packages/redis/package.json +++ b/packages/redis/package.json @@ -8,7 +8,7 @@ }, "dependencies": { "@openpanel/json": "workspace:*", - "ioredis": "^5.7.0" + "ioredis": "5.8.2" }, "devDependencies": { "@openpanel/db": "workspace:*", diff --git a/packages/trpc/package.json b/packages/trpc/package.json index 630e74d9..6632ce43 100644 --- a/packages/trpc/package.json +++ b/packages/trpc/package.json @@ -16,6 +16,7 @@ "@openpanel/payments": "workspace:^", "@openpanel/redis": "workspace:*", "@openpanel/validation": "workspace:*", + "@openpanel/queue": "workspace:*", "@trpc-limiter/redis": "^0.0.2", "@trpc/client": "^11.6.0", "@trpc/server": "^11.6.0", diff --git a/packages/trpc/src/root.ts b/packages/trpc/src/root.ts index 2d788837..d852c9c1 100644 --- a/packages/trpc/src/root.ts +++ b/packages/trpc/src/root.ts @@ -4,6 +4,7 @@ import { chatRouter } from './routers/chat'; import { clientRouter } from './routers/client'; import { dashboardRouter } from './routers/dashboard'; import { eventRouter } from './routers/event'; +import { importRouter } from './routers/import'; import { integrationRouter } from './routers/integration'; import { notificationRouter } from './routers/notification'; import { onboardingRouter } from './routers/onboarding'; @@ -40,6 +41,7 @@ export const appRouter = createTRPCRouter({ reference: referenceRouter, notification: notificationRouter, integration: integrationRouter, + import: importRouter, auth: authRouter, subscription: subscriptionRouter, overview: overviewRouter, diff --git a/packages/trpc/src/routers/auth.ts b/packages/trpc/src/routers/auth.ts index 34a9d19c..ccfc7f7b 100644 --- a/packages/trpc/src/routers/auth.ts +++ b/packages/trpc/src/routers/auth.ts @@ -12,7 +12,7 @@ import { validateSessionToken, verifyPasswordHash, } from '@openpanel/auth'; -import { generateSecureId } from '@openpanel/common/server/id'; +import { generateSecureId } from '@openpanel/common/server'; import { connectUserToOrganization, db, diff --git a/packages/trpc/src/routers/import.ts b/packages/trpc/src/routers/import.ts new file mode 100644 index 00000000..d77effa1 --- /dev/null +++ b/packages/trpc/src/routers/import.ts @@ -0,0 +1,178 @@ +import { z } from 'zod'; + +import { db } from '@openpanel/db'; +import { importQueue } from '@openpanel/queue'; +import { zCreateImport } from '@openpanel/validation'; + +import { getProjectAccess } from '../access'; +import { TRPCAccessError } from '../errors'; +import { createTRPCRouter, protectedProcedure } from '../trpc'; + +export const importRouter = createTRPCRouter({ + list: protectedProcedure + .input(z.object({ projectId: z.string() })) + .query(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + + if (!access) { + throw TRPCAccessError('You do not have access to this project'); + } + + return db.import.findMany({ + where: { + projectId: input.projectId, + }, + orderBy: { + createdAt: 'desc', + }, + }); + }), + + get: protectedProcedure + .input(z.object({ id: z.string() })) + .query(async ({ input, ctx }) => { + const importRecord = await db.import.findUniqueOrThrow({ + where: { + id: input.id, + }, + include: { + project: true, + }, + }); + + const access = await getProjectAccess({ + projectId: importRecord.projectId, + userId: ctx.session.userId, + }); + + if (!access) { + throw TRPCAccessError('You do not have access to this import'); + } + + return importRecord; + }), + + create: protectedProcedure + .input(zCreateImport) + .mutation(async ({ input, ctx }) => { + const access = await getProjectAccess({ + projectId: input.projectId, + userId: ctx.session.userId, + }); + + if (!access || (typeof access !== 'boolean' && access.level === 'read')) { + throw TRPCAccessError( + 'You do not have permission to create imports for this project', + ); + } + + // Create import record + const importRecord = await db.import.create({ + data: { + projectId: input.projectId, + config: input.config, + status: 'pending', + }, + }); + + // Add job to queue + const job = await importQueue.add('import', { + type: 'import', + payload: { + importId: importRecord.id, + }, + }); + + // Update import record with job ID + await db.import.update({ + where: { id: importRecord.id }, + data: { jobId: job.id }, + }); + + return { + ...importRecord, + jobId: job.id, + }; + }), + + delete: protectedProcedure + .input(z.object({ id: z.string() })) + .mutation(async ({ input, ctx }) => { + const importRecord = await db.import.findUniqueOrThrow({ + where: { + id: input.id, + }, + }); + + const access = await getProjectAccess({ + projectId: importRecord.projectId, + userId: ctx.session.userId, + }); + + if (!access || (typeof access !== 'boolean' && access.level === 'read')) { + throw TRPCAccessError( + 'You do not have permission to delete imports for this project', + ); + } + + if (importRecord.jobId) { + const job = await importQueue.getJob(importRecord.jobId); + if (job) { + await job.remove(); + } + } + + return db.import.delete({ + where: { + id: input.id, + }, + }); + }), + + retry: protectedProcedure + .input(z.object({ id: z.string() })) + .mutation(async ({ input, ctx }) => { + const importRecord = await db.import.findUniqueOrThrow({ + where: { + id: input.id, + }, + }); + + const access = await getProjectAccess({ + projectId: importRecord.projectId, + userId: ctx.session.userId, + }); + + if (!access || (typeof access !== 'boolean' && access.level === 'read')) { + throw TRPCAccessError( + 'You do not have permission to retry imports for this project', + ); + } + + // Only allow retry for failed imports + if (importRecord.status !== 'failed') { + throw new Error('Only failed imports can be retried'); + } + + // Add new job to queue + const job = await importQueue.add('import', { + type: 'import', + payload: { + importId: importRecord.id, + }, + }); + + // Update import record + return db.import.update({ + where: { id: importRecord.id }, + data: { + jobId: job.id, + status: 'pending', + errorMessage: null, + }, + }); + }), +}); diff --git a/packages/trpc/src/routers/organization.ts b/packages/trpc/src/routers/organization.ts index 21bf029a..40ff4ea1 100644 --- a/packages/trpc/src/routers/organization.ts +++ b/packages/trpc/src/routers/organization.ts @@ -11,7 +11,7 @@ import { } from '@openpanel/db'; import { zEditOrganization, zInviteUser } from '@openpanel/validation'; -import { generateSecureId } from '@openpanel/common/server/id'; +import { generateSecureId } from '@openpanel/common/server'; import { sendEmail } from '@openpanel/email'; import { addDays } from 'date-fns'; import { getOrganizationAccess } from '../access'; diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts index 44ae45de..0b1cefe6 100644 --- a/packages/validation/src/index.ts +++ b/packages/validation/src/index.ts @@ -435,3 +435,54 @@ export const zEditOrganization = z.object({ name: z.string().min(2), timezone: z.string().min(1), }); + +const zProjectMapper = z.object({ + from: z.string().min(1), + to: z.string().min(1), +}); + +const createFileImportConfig = (provider: T) => + z.object({ + provider: z.literal(provider), + type: z.literal('file'), + fileUrl: z.string().url(), + }); + +// Import configs +export const zUmamiImportConfig = createFileImportConfig('umami').extend({ + projectMapper: z.array(zProjectMapper), +}); + +export type IUmamiImportConfig = z.infer; + +export const zPlausibleImportConfig = createFileImportConfig('plausible'); +export type IPlausibleImportConfig = z.infer; + +export const zMixpanelImportConfig = z.object({ + provider: z.literal('mixpanel'), + type: z.literal('api'), + serviceAccount: z.string().min(1), + serviceSecret: z.string().min(1), + projectId: z.string().min(1), + from: z.string().min(1), + to: z.string().min(1), + mapScreenViewProperty: z.string().optional(), +}); +export type IMixpanelImportConfig = z.infer; + +export type IImportConfig = + | IUmamiImportConfig + | IPlausibleImportConfig + | IMixpanelImportConfig; + +export const zCreateImport = z.object({ + projectId: z.string().min(1), + provider: z.enum(['umami', 'plausible', 'mixpanel']), + config: z.union([ + zUmamiImportConfig, + zPlausibleImportConfig, + zMixpanelImportConfig, + ]), +}); + +export type ICreateImport = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f15819bd..6b6e6377 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -226,7 +226,7 @@ importers: specifier: ^4.1.0 version: 4.1.0 tsdown: - specifier: ^0.14.2 + specifier: 0.14.2 version: 0.14.2(typescript@5.9.3) typescript: specifier: 'catalog:' @@ -373,6 +373,9 @@ importers: '@openpanel/constants': specifier: workspace:^ version: link:../../packages/constants + '@openpanel/importer': + specifier: workspace:^ + version: link:../../packages/importer '@openpanel/integrations': specifier: workspace:^ version: link:../../packages/integrations @@ -815,6 +818,9 @@ importers: '@openpanel/email': specifier: workspace:* version: link:../../packages/email + '@openpanel/importer': + specifier: workspace:* + version: link:../../packages/importer '@openpanel/integrations': specifier: workspace:^ version: link:../../packages/integrations @@ -874,7 +880,7 @@ importers: specifier: ^9.0.8 version: 9.0.8 tsdown: - specifier: ^0.14.2 + specifier: 0.14.2 version: 0.14.2(typescript@5.9.3) typescript: specifier: 'catalog:' @@ -923,58 +929,6 @@ importers: specifier: 'catalog:' version: 5.9.3 - packages/cli: - dependencies: - '@openpanel/common': - specifier: workspace:* - version: link:../common - arg: - specifier: ^5.0.2 - version: 5.0.2 - glob: - specifier: ^10.4.3 - version: 10.4.5 - inquirer: - specifier: ^9.3.5 - version: 9.3.6 - p-limit: - specifier: ^6.1.0 - version: 6.1.0 - progress: - specifier: ^2.0.3 - version: 2.0.3 - ramda: - specifier: ^0.29.1 - version: 0.29.1 - zod: - specifier: 'catalog:' - version: 3.24.2 - devDependencies: - '@openpanel/db': - specifier: workspace:^ - version: link:../db - '@openpanel/sdk': - specifier: workspace:* - version: link:../sdks/sdk - '@openpanel/tsconfig': - specifier: workspace:* - version: link:../../tooling/typescript - '@types/node': - specifier: 'catalog:' - version: 24.7.1 - '@types/progress': - specifier: ^2.0.7 - version: 2.0.7 - '@types/ramda': - specifier: ^0.30.1 - version: 0.30.1 - tsup: - specifier: ^7.2.0 - version: 7.3.0(postcss@8.5.6)(typescript@5.9.3) - typescript: - specifier: 'catalog:' - version: 5.9.3 - packages/common: dependencies: '@openpanel/constants': @@ -1048,8 +1002,8 @@ importers: packages/db: dependencies: '@clickhouse/client': - specifier: ^1.2.0 - version: 1.2.0 + specifier: ^1.12.1 + version: 1.12.1 '@openpanel/common': specifier: workspace:* version: link:../common @@ -1186,6 +1140,55 @@ importers: specifier: 'catalog:' version: 5.9.3 + packages/importer: + dependencies: + '@openpanel/common': + specifier: workspace:* + version: link:../common + '@openpanel/db': + specifier: workspace:* + version: link:../db + '@openpanel/queue': + specifier: workspace:* + version: link:../queue + '@openpanel/validation': + specifier: workspace:* + version: link:../validation + csv-parse: + specifier: ^6.1.0 + version: 6.1.0 + ramda: + specifier: ^0.29.1 + version: 0.29.1 + uuid: + specifier: ^9.0.1 + version: 9.0.1 + zod: + specifier: 'catalog:' + version: 3.24.2 + devDependencies: + '@openpanel/logger': + specifier: workspace:* + version: link:../logger + '@types/node': + specifier: ^20.0.0 + version: 20.14.8 + '@types/ramda': + specifier: ^0.31.1 + version: 0.31.1 + '@types/uuid': + specifier: ^9.0.7 + version: 9.0.8 + bullmq: + specifier: ^5.8.7 + version: 5.8.7 + typescript: + specifier: ^5.0.0 + version: 5.9.3 + vitest: + specifier: ^1.0.0 + version: 1.6.1(@types/node@20.14.8)(jsdom@26.1.0)(lightningcss@1.30.1)(terser@5.27.1) + packages/integrations: dependencies: '@slack/bolt': @@ -1320,8 +1323,8 @@ importers: specifier: workspace:* version: link:../json ioredis: - specifier: ^5.7.0 - version: 5.7.0 + specifier: 5.8.2 + version: 5.8.2 devDependencies: '@openpanel/db': specifier: workspace:* @@ -1509,6 +1512,9 @@ importers: '@openpanel/payments': specifier: workspace:^ version: link:../payments + '@openpanel/queue': + specifier: workspace:* + version: link:../queue '@openpanel/redis': specifier: workspace:* version: link:../redis @@ -2710,9 +2716,16 @@ packages: '@capsizecss/unpack@2.4.0': resolution: {integrity: sha512-GrSU71meACqcmIUxPYOJvGKF0yryjN/L1aCuE9DViCTJI7bfkjgYDPD1zbNDcINJwSSP6UaBZY9GAbYDO7re0Q==} + '@clickhouse/client-common@1.12.1': + resolution: {integrity: sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==} + '@clickhouse/client-common@1.2.0': resolution: {integrity: sha512-VfA/C/tVJ2eNe72CaQ7eXmai+yqFEvZjQZiNtvJoOMLP+Vtb6DzqH9nfkgsiHHMhUhhclvt2mFh6+euk1Ea5wA==} + '@clickhouse/client@1.12.1': + resolution: {integrity: sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==} + engines: {node: '>=16'} + '@clickhouse/client@1.2.0': resolution: {integrity: sha512-zMp2EhMfp1IrFKr/NjDwNiLsf7nq68nW8lGKszwFe7Iglc6Z5PY9ZA9Hd0XqAk75Q1NmFrkGCP1r3JCM1Nm1Bw==} engines: {node: '>=16'} @@ -2866,6 +2879,12 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.21.5': + resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + '@esbuild/aix-ppc64@0.24.0': resolution: {integrity: sha512-WtKdFM7ls47zkKHFVzMz8opM7LkcsIp9amDUBIAWirg70RM71WRSjdILPsY5Uv1D42ZpUfaPILDlfactHgsRkw==} engines: {node: '>=18'} @@ -2908,6 +2927,12 @@ packages: cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.21.5': + resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm64@0.24.0': resolution: {integrity: sha512-Vsm497xFM7tTIPYK9bNTYJyF/lsP590Qc1WxJdlB6ljCbdZKU9SY8i7+Iin4kyhV/KV5J2rOKsBQbB77Ab7L/w==} engines: {node: '>=18'} @@ -2950,6 +2975,12 @@ packages: cpu: [arm] os: [android] + '@esbuild/android-arm@0.21.5': + resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + '@esbuild/android-arm@0.24.0': resolution: {integrity: sha512-arAtTPo76fJ/ICkXWetLCc9EwEHKaeya4vMrReVlEIUCAUncH7M4bhMQ+M9Vf+FFOZJdTNMXNBrWwW+OXWpSew==} engines: {node: '>=18'} @@ -2992,6 +3023,12 @@ packages: cpu: [x64] os: [android] + '@esbuild/android-x64@0.21.5': + resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + '@esbuild/android-x64@0.24.0': resolution: {integrity: sha512-t8GrvnFkiIY7pa7mMgJd7p8p8qqYIz1NYiAoKc75Zyv73L3DZW++oYMSHPRarcotTKuSs6m3hTOa5CKHaS02TQ==} engines: {node: '>=18'} @@ -3034,6 +3071,12 @@ packages: cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.21.5': + resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-arm64@0.24.0': resolution: {integrity: sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==} engines: {node: '>=18'} @@ -3076,6 +3119,12 @@ packages: cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.21.5': + resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + '@esbuild/darwin-x64@0.24.0': resolution: {integrity: sha512-rgtz6flkVkh58od4PwTRqxbKH9cOjaXCMZgWD905JOzjFKW+7EiUObfd/Kav+A6Gyud6WZk9w+xu6QLytdi2OA==} engines: {node: '>=18'} @@ -3118,6 +3167,12 @@ packages: cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.21.5': + resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-arm64@0.24.0': resolution: {integrity: sha512-6Mtdq5nHggwfDNLAHkPlyLBpE5L6hwsuXZX8XNmHno9JuL2+bg2BX5tRkwjyfn6sKbxZTq68suOjgWqCicvPXA==} engines: {node: '>=18'} @@ -3160,6 +3215,12 @@ packages: cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.21.5': + resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + '@esbuild/freebsd-x64@0.24.0': resolution: {integrity: sha512-D3H+xh3/zphoX8ck4S2RxKR6gHlHDXXzOf6f/9dbFt/NRBDIE33+cVa49Kil4WUjxMGW0ZIYBYtaGCa2+OsQwQ==} engines: {node: '>=18'} @@ -3202,6 +3263,12 @@ packages: cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.21.5': + resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm64@0.24.0': resolution: {integrity: sha512-TDijPXTOeE3eaMkRYpcy3LarIg13dS9wWHRdwYRnzlwlA370rNdZqbcp0WTyyV/k2zSxfko52+C7jU5F9Tfj1g==} engines: {node: '>=18'} @@ -3244,6 +3311,12 @@ packages: cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.21.5': + resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + '@esbuild/linux-arm@0.24.0': resolution: {integrity: sha512-gJKIi2IjRo5G6Glxb8d3DzYXlxdEj2NlkixPsqePSZMhLudqPhtZ4BUrpIuTjJYXxvF9njql+vRjB2oaC9XpBw==} engines: {node: '>=18'} @@ -3286,6 +3359,12 @@ packages: cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.21.5': + resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-ia32@0.24.0': resolution: {integrity: sha512-K40ip1LAcA0byL05TbCQ4yJ4swvnbzHscRmUilrmP9Am7//0UjPreh4lpYzvThT2Quw66MhjG//20mrufm40mA==} engines: {node: '>=18'} @@ -3328,6 +3407,12 @@ packages: cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.21.5': + resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-loong64@0.24.0': resolution: {integrity: sha512-0mswrYP/9ai+CU0BzBfPMZ8RVm3RGAN/lmOMgW4aFUSOQBjA31UP8Mr6DDhWSuMwj7jaWOT0p0WoZ6jeHhrD7g==} engines: {node: '>=18'} @@ -3370,6 +3455,12 @@ packages: cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.21.5': + resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-mips64el@0.24.0': resolution: {integrity: sha512-hIKvXm0/3w/5+RDtCJeXqMZGkI2s4oMUGj3/jM0QzhgIASWrGO5/RlzAzm5nNh/awHE0A19h/CvHQe6FaBNrRA==} engines: {node: '>=18'} @@ -3412,6 +3503,12 @@ packages: cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.21.5': + resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-ppc64@0.24.0': resolution: {integrity: sha512-HcZh5BNq0aC52UoocJxaKORfFODWXZxtBaaZNuN3PUX3MoDsChsZqopzi5UupRhPHSEHotoiptqikjN/B77mYQ==} engines: {node: '>=18'} @@ -3454,6 +3551,12 @@ packages: cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.21.5': + resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-riscv64@0.24.0': resolution: {integrity: sha512-bEh7dMn/h3QxeR2KTy1DUszQjUrIHPZKyO6aN1X4BCnhfYhuQqedHaa5MxSQA/06j3GpiIlFGSsy1c7Gf9padw==} engines: {node: '>=18'} @@ -3496,6 +3599,12 @@ packages: cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.21.5': + resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-s390x@0.24.0': resolution: {integrity: sha512-ZcQ6+qRkw1UcZGPyrCiHHkmBaj9SiCD8Oqd556HldP+QlpUIe2Wgn3ehQGVoPOvZvtHm8HPx+bH20c9pvbkX3g==} engines: {node: '>=18'} @@ -3538,6 +3647,12 @@ packages: cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.21.5': + resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + '@esbuild/linux-x64@0.24.0': resolution: {integrity: sha512-vbutsFqQ+foy3wSSbmjBXXIJ6PL3scghJoM8zCL142cGaZKAdCZHyf+Bpu/MmX9zT9Q0zFBVKb36Ma5Fzfa8xA==} engines: {node: '>=18'} @@ -3604,6 +3719,12 @@ packages: cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.21.5': + resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + '@esbuild/netbsd-x64@0.24.0': resolution: {integrity: sha512-hjQ0R/ulkO8fCYFsG0FZoH+pWgTTDreqpqY7UnQntnaKv95uP5iW3+dChxnx7C3trQQU40S+OgWhUVwCjVFLvg==} engines: {node: '>=18'} @@ -3676,6 +3797,12 @@ packages: cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.21.5': + resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + '@esbuild/openbsd-x64@0.24.0': resolution: {integrity: sha512-4ir0aY1NGUhIC1hdoCzr1+5b43mw99uNwVzhIq1OY3QcEwPDO3B7WNXBzaKY5Nsf1+N11i1eOfFcq+D/gOS15Q==} engines: {node: '>=18'} @@ -3730,6 +3857,12 @@ packages: cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.21.5': + resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + '@esbuild/sunos-x64@0.24.0': resolution: {integrity: sha512-jVzdzsbM5xrotH+W5f1s+JtUy1UWgjU0Cf4wMvffTB8m6wP5/kx0KiaLHlbJO+dMgtxKV8RQ/JvtlFcdZ1zCPA==} engines: {node: '>=18'} @@ -3772,6 +3905,12 @@ packages: cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.21.5': + resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-arm64@0.24.0': resolution: {integrity: sha512-iKc8GAslzRpBytO2/aN3d2yb2z8XTVfNV0PjGlCxKo5SgWmNXx82I/Q3aG1tFfS+A2igVCY97TJ8tnYwpUWLCA==} engines: {node: '>=18'} @@ -3814,6 +3953,12 @@ packages: cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.21.5': + resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-ia32@0.24.0': resolution: {integrity: sha512-vQW36KZolfIudCcTnaTpmLQ24Ha1RjygBo39/aLkM2kmjkWmZGEJ5Gn9l5/7tzXA42QGIoWbICfg6KLLkIw6yw==} engines: {node: '>=18'} @@ -3856,6 +4001,12 @@ packages: cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.21.5': + resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + '@esbuild/win32-x64@0.24.0': resolution: {integrity: sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==} engines: {node: '>=18'} @@ -4178,9 +4329,6 @@ packages: resolution: {integrity: sha512-R7Gsg6elpuqdn55fBH2y9oYzrU/yKrSmIsDX4ROT51vohrECFzTf2zw9BfUbOW8xjfmM2QbVoVYdTwhrtEKWSQ==} engines: {node: '>=18'} - '@ioredis/commands@1.3.0': - resolution: {integrity: sha512-M/T6Zewn7sDaBQEqIZ8Rb+i9y8qfGmq+5SDFSf9sA2lUZTmdDLVdOiQaeDp+Q4wElZ9HG1GAX5KhDaidp6LQsQ==} - '@ioredis/commands@1.4.0': resolution: {integrity: sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==} @@ -8570,9 +8718,6 @@ packages: '@types/node@20.14.8': resolution: {integrity: sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==} - '@types/node@22.18.0': - resolution: {integrity: sha512-m5ObIqwsUp6BZzyiy4RdZpzWGub9bqLJMvZDD0QMXhxjqMHMENlj+SqF5QxoUwaQNFe+8kz8XM8ZQhqkQPTgMQ==} - '@types/node@24.7.1': resolution: {integrity: sha512-CmyhGZanP88uuC5GpWU9q+fI61j2SkhO3UGMUdfYRE6Bcy0ccyzn1Rqj9YAB/ZY4kOXmNf0ocah5GtphmLMP6Q==} @@ -8588,9 +8733,6 @@ packages: '@types/pg@8.6.1': resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==} - '@types/progress@2.0.7': - resolution: {integrity: sha512-iadjw02vte8qWx7U0YM++EybBha2CQLPGu9iJ97whVgJUT5Zq9MjAPYUnbfRI2Kpehimf1QjFJYxD0t8nqzu5w==} - '@types/promise.allsettled@1.0.6': resolution: {integrity: sha512-wA0UT0HeT2fGHzIFV9kWpYz5mdoyLxKrTgMdZQM++5h6pYAFH73HXcQhefg24nD1yivUFEn5KU+EF4b+CXJ4Wg==} @@ -8603,15 +8745,15 @@ packages: '@types/ramda@0.29.10': resolution: {integrity: sha512-0BzWVKtSEtignlk+XBuK88Il5wzQwbRVfEkzE8iKm02NYHMGQ/9ffB05M+zXhTCqo50DOIAT9pNSJsjFMMG4rQ==} - '@types/ramda@0.30.1': - resolution: {integrity: sha512-aoyF/ADPL6N+/NXXfhPWF+Qj6w1Cql59m9wX0Gi15uyF+bpzXeLd63HPdiTDE2bmLXfNcVufsDPKmbfOrOzTBA==} - '@types/ramda@0.30.2': resolution: {integrity: sha512-PyzHvjCalm2BRYjAU6nIB3TprYwMNOUY/7P/N8bSzp9W/yM2YrtGtAnnVtaCNSeOZ8DzKyFDvaqQs7LnWwwmBA==} '@types/ramda@0.31.0': resolution: {integrity: sha512-1lWWZ/2YiNttGcIUxQwnvMuh55GIEbn/zlpzzEojAsbxquI/TXQZCRaXsfxG1CHjlqGoqxWePkvaM/5qYHNuvQ==} + '@types/ramda@0.31.1': + resolution: {integrity: sha512-Vt6sFXnuRpzaEj+yeutA0q3bcAsK7wdPuASIzR9LXqL4gJPyFw8im9qchlbp4ltuf3kDEIRmPJTD/Fkg60dn7g==} + '@types/range-parser@1.2.7': resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} @@ -8734,6 +8876,9 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 + '@vitest/expect@1.6.1': + resolution: {integrity: sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog==} + '@vitest/expect@3.1.3': resolution: {integrity: sha512-7FTQQuuLKmN1Ig/h+h/GO+44Q1IlglPlR2es4ab7Yvfx+Uk5xsv+Ykk+MEt/M2Yn/xGmzaLKxGw2lgy2bwuYqg==} @@ -8751,15 +8896,27 @@ packages: '@vitest/pretty-format@3.1.3': resolution: {integrity: sha512-i6FDiBeJUGLDKADw2Gb01UtUNb12yyXAqC/mmRWuYl+m/U9GS7s8us5ONmGkGpUUo7/iAYzI2ePVfOZTYvUifA==} + '@vitest/runner@1.6.1': + resolution: {integrity: sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA==} + '@vitest/runner@3.1.3': resolution: {integrity: sha512-Tae+ogtlNfFei5DggOsSUvkIaSuVywujMj6HzR97AHK6XK8i3BuVyIifWAm/sE3a15lF5RH9yQIrbXYuo0IFyA==} + '@vitest/snapshot@1.6.1': + resolution: {integrity: sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ==} + '@vitest/snapshot@3.1.3': resolution: {integrity: sha512-XVa5OPNTYUsyqG9skuUkFzAeFnEzDp8hQu7kZ0N25B1+6KjGm4hWLtURyBbsIAOekfWQ7Wuz/N/XXzgYO3deWQ==} + '@vitest/spy@1.6.1': + resolution: {integrity: sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw==} + '@vitest/spy@3.1.3': resolution: {integrity: sha512-x6w+ctOEmEXdWaa6TO4ilb7l9DxPR5bwEb6hILKuxfU1NqWT2mpJD9NJN7t3OTfxmVlOMrvtoFJGdgyzZ605lQ==} + '@vitest/utils@1.6.1': + resolution: {integrity: sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g==} + '@vitest/utils@3.1.3': resolution: {integrity: sha512-2Ltrpht4OmHO9+c/nmHtF09HWiyWdworqnHIwjfvDyWjuwKbdkcS9AnhsDn+8E2RM4x++foD1/tNuLPVvWG1Rg==} @@ -8920,10 +9077,6 @@ packages: resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} engines: {node: '>=12'} - ansis@4.1.0: - resolution: {integrity: sha512-BGcItUBWSMRgOCe+SVZJ+S7yTRG0eGt9cXAHev72yuGcY23hnLA7Bky5L/xLyPINoSN95geovfBkqoTlNZYa7w==} - engines: {node: '>=14'} - ansis@4.2.0: resolution: {integrity: sha512-HqZ5rWlFjGiV0tDm3UxxgNRqsOTniqoKZu0pIAfh7TZQMGuZK+hH0drySty0si0QXj1ieop4+SkSfPZBPPkHig==} engines: {node: '>=14'} @@ -9009,6 +9162,9 @@ packages: asap@2.0.6: resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} + assertion-error@1.1.0: + resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -9399,6 +9555,10 @@ packages: ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + chai@4.5.0: + resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} + engines: {node: '>=4'} + chai@5.2.0: resolution: {integrity: sha512-mCuXncKXk5iCLhfhwTc0izo0gtEmpz5CtG2y8GiOINBlMVS6v8TMRc5TaLWKS6692m9+dVVfzgeVxR5UxWHTYw==} engines: {node: '>=12'} @@ -9442,6 +9602,9 @@ packages: charenc@0.0.2: resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + check-error@1.0.3: + resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} + check-error@2.1.1: resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} engines: {node: '>= 16'} @@ -9836,6 +9999,9 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + csv-parse@6.1.0: + resolution: {integrity: sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==} + d3-array@2.12.1: resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==} @@ -10123,6 +10289,10 @@ packages: decode-named-character-reference@1.0.2: resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==} + deep-eql@4.1.4: + resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==} + engines: {node: '>=6'} + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} @@ -10245,6 +10415,10 @@ packages: diff-match-patch@1.0.5: resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} + diff-sequences@29.6.3: + resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + diff@5.2.0: resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==} engines: {node: '>=0.3.1'} @@ -10525,6 +10699,11 @@ packages: engines: {node: '>=12'} hasBin: true + esbuild@0.21.5: + resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} + engines: {node: '>=12'} + hasBin: true + esbuild@0.24.0: resolution: {integrity: sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==} engines: {node: '>=18'} @@ -11122,6 +11301,9 @@ packages: resolution: {integrity: sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==} engines: {node: '>=18'} + get-func-name@2.0.2: + resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} + get-intrinsic@1.2.4: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} @@ -11594,10 +11776,6 @@ packages: invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} - ioredis@5.7.0: - resolution: {integrity: sha512-NUcA93i1lukyXU+riqEyPtSEkyFq8tX90uL659J+qpCZ3rEdViB/APC58oAhIh3+bJln2hzdlZbBZsGNrlsR8g==} - engines: {node: '>=12.22.0'} - ioredis@5.8.2: resolution: {integrity: sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==} engines: {node: '>=12.22.0'} @@ -12269,6 +12447,10 @@ packages: resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + local-pkg@0.5.1: + resolution: {integrity: sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ==} + engines: {node: '>=14'} + local-pkg@1.1.2: resolution: {integrity: sha512-arhlxbFRmoQHl33a0Zkle/YWlmNwoyt6QNZEIJcqNbdrsix5Lvc4HyyI3EnwxTYlZYc32EbYrQ8SzEZ7dqgg9A==} engines: {node: '>=14'} @@ -12378,6 +12560,9 @@ packages: lottie-web@5.12.2: resolution: {integrity: sha512-uvhvYPC8kGPjXT3MyKMrL3JitEAmDMp30lVkuq/590Mw9ok6pWcFCwXJveo0t5uqYw1UREQHofD+jVpdjBv8wg==} + loupe@2.3.7: + resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} + loupe@3.1.3: resolution: {integrity: sha512-kkIp7XSkP78ZxJEsSxW3712C6teJVoeHHwgo9zJ380de7IYyJ2ISlxojcH2pC5OFLewESmnRi/+XCDIEEVyoug==} @@ -13329,8 +13514,8 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} - p-limit@6.1.0: - resolution: {integrity: sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==} + p-limit@5.0.0: + resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==} engines: {node: '>=18'} p-limit@6.2.0: @@ -13498,6 +13683,9 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pathval@1.1.1: + resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} + pathval@2.0.0: resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==} engines: {node: '>= 14.16'} @@ -15107,6 +15295,9 @@ packages: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} + strip-literal@2.1.1: + resolution: {integrity: sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==} + strip-literal@3.1.0: resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} @@ -15374,6 +15565,10 @@ packages: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} + tinypool@0.8.4: + resolution: {integrity: sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==} + engines: {node: '>=14.0.0'} + tinypool@1.0.2: resolution: {integrity: sha512-al6n+QEANGFOMf/dmUMsuS5/r9B06uwlyNjZZql/zv8J7ybHCgoihBNORZCY2mzUuAnomQa2JdhyHKzZxPCrFA==} engines: {node: ^18.0.0 || >=20.0.0} @@ -15382,6 +15577,10 @@ packages: resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} engines: {node: '>=14.0.0'} + tinyspy@2.2.1: + resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==} + engines: {node: '>=14.0.0'} + tinyspy@3.0.2: resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} engines: {node: '>=14.0.0'} @@ -15528,6 +15727,10 @@ packages: resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==} engines: {node: '>=4'} + type-detect@4.1.0: + resolution: {integrity: sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==} + engines: {node: '>=4'} + type-fest@0.16.0: resolution: {integrity: sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg==} engines: {node: '>=10'} @@ -15636,9 +15839,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.21.0: - resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - undici-types@7.14.0: resolution: {integrity: sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==} @@ -16024,6 +16224,11 @@ packages: victory-vendor@36.9.1: resolution: {integrity: sha512-+pZIP+U3pEJdDCeFmsXwHzV7vNHQC/eIbHklfe2ZCZqayYRH7lQbHcVgsJ0XOOv27hWs4jH4MONgXxHMObTMSA==} + vite-node@1.6.1: + resolution: {integrity: sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + vite-node@3.1.3: resolution: {integrity: sha512-uHV4plJ2IxCl4u1up1FQRrqclylKAogbtBfOTwcuJ28xFi+89PZ57BRh+naIRvH70HPwxy5QHYzg1OrEaC7AbA==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -16037,6 +16242,37 @@ packages: vite: optional: true + vite@5.4.21: + resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || >=20.0.0 + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + vite@6.3.3: resolution: {integrity: sha512-5nXH+QsELbFKhsEfWLkHrvgRpTdGJzqOZ+utSdmPTvwHmvU6ITTm3xx+mRusihkcI8GeC7lCDyn3kDtiki9scw==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -16173,6 +16409,31 @@ packages: vite: optional: true + vitest@1.6.1: + resolution: {integrity: sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': 1.6.1 + '@vitest/ui': 1.6.1 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + vitest@3.1.3: resolution: {integrity: sha512-188iM4hAHQ0km23TN/adso1q5hhwKqUpv+Sd6p5sOuh6FhQnRNW3IsiIpvxqahtBabsJ2SLZgmGSpcYK4wQYJw==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -17931,8 +18192,14 @@ snapshots: transitivePeerDependencies: - encoding + '@clickhouse/client-common@1.12.1': {} + '@clickhouse/client-common@1.2.0': {} + '@clickhouse/client@1.12.1': + dependencies: + '@clickhouse/client-common': 1.12.1 + '@clickhouse/client@1.2.0': dependencies: '@clickhouse/client-common': 1.2.0 @@ -18084,6 +18351,9 @@ snapshots: '@esbuild/aix-ppc64@0.19.12': optional: true + '@esbuild/aix-ppc64@0.21.5': + optional: true + '@esbuild/aix-ppc64@0.24.0': optional: true @@ -18105,6 +18375,9 @@ snapshots: '@esbuild/android-arm64@0.19.12': optional: true + '@esbuild/android-arm64@0.21.5': + optional: true + '@esbuild/android-arm64@0.24.0': optional: true @@ -18126,6 +18399,9 @@ snapshots: '@esbuild/android-arm@0.19.12': optional: true + '@esbuild/android-arm@0.21.5': + optional: true + '@esbuild/android-arm@0.24.0': optional: true @@ -18147,6 +18423,9 @@ snapshots: '@esbuild/android-x64@0.19.12': optional: true + '@esbuild/android-x64@0.21.5': + optional: true + '@esbuild/android-x64@0.24.0': optional: true @@ -18168,6 +18447,9 @@ snapshots: '@esbuild/darwin-arm64@0.19.12': optional: true + '@esbuild/darwin-arm64@0.21.5': + optional: true + '@esbuild/darwin-arm64@0.24.0': optional: true @@ -18189,6 +18471,9 @@ snapshots: '@esbuild/darwin-x64@0.19.12': optional: true + '@esbuild/darwin-x64@0.21.5': + optional: true + '@esbuild/darwin-x64@0.24.0': optional: true @@ -18210,6 +18495,9 @@ snapshots: '@esbuild/freebsd-arm64@0.19.12': optional: true + '@esbuild/freebsd-arm64@0.21.5': + optional: true + '@esbuild/freebsd-arm64@0.24.0': optional: true @@ -18231,6 +18519,9 @@ snapshots: '@esbuild/freebsd-x64@0.19.12': optional: true + '@esbuild/freebsd-x64@0.21.5': + optional: true + '@esbuild/freebsd-x64@0.24.0': optional: true @@ -18252,6 +18543,9 @@ snapshots: '@esbuild/linux-arm64@0.19.12': optional: true + '@esbuild/linux-arm64@0.21.5': + optional: true + '@esbuild/linux-arm64@0.24.0': optional: true @@ -18273,6 +18567,9 @@ snapshots: '@esbuild/linux-arm@0.19.12': optional: true + '@esbuild/linux-arm@0.21.5': + optional: true + '@esbuild/linux-arm@0.24.0': optional: true @@ -18294,6 +18591,9 @@ snapshots: '@esbuild/linux-ia32@0.19.12': optional: true + '@esbuild/linux-ia32@0.21.5': + optional: true + '@esbuild/linux-ia32@0.24.0': optional: true @@ -18315,6 +18615,9 @@ snapshots: '@esbuild/linux-loong64@0.19.12': optional: true + '@esbuild/linux-loong64@0.21.5': + optional: true + '@esbuild/linux-loong64@0.24.0': optional: true @@ -18336,6 +18639,9 @@ snapshots: '@esbuild/linux-mips64el@0.19.12': optional: true + '@esbuild/linux-mips64el@0.21.5': + optional: true + '@esbuild/linux-mips64el@0.24.0': optional: true @@ -18357,6 +18663,9 @@ snapshots: '@esbuild/linux-ppc64@0.19.12': optional: true + '@esbuild/linux-ppc64@0.21.5': + optional: true + '@esbuild/linux-ppc64@0.24.0': optional: true @@ -18378,6 +18687,9 @@ snapshots: '@esbuild/linux-riscv64@0.19.12': optional: true + '@esbuild/linux-riscv64@0.21.5': + optional: true + '@esbuild/linux-riscv64@0.24.0': optional: true @@ -18399,6 +18711,9 @@ snapshots: '@esbuild/linux-s390x@0.19.12': optional: true + '@esbuild/linux-s390x@0.21.5': + optional: true + '@esbuild/linux-s390x@0.24.0': optional: true @@ -18420,6 +18735,9 @@ snapshots: '@esbuild/linux-x64@0.19.12': optional: true + '@esbuild/linux-x64@0.21.5': + optional: true + '@esbuild/linux-x64@0.24.0': optional: true @@ -18453,6 +18771,9 @@ snapshots: '@esbuild/netbsd-x64@0.19.12': optional: true + '@esbuild/netbsd-x64@0.21.5': + optional: true + '@esbuild/netbsd-x64@0.24.0': optional: true @@ -18489,6 +18810,9 @@ snapshots: '@esbuild/openbsd-x64@0.19.12': optional: true + '@esbuild/openbsd-x64@0.21.5': + optional: true + '@esbuild/openbsd-x64@0.24.0': optional: true @@ -18516,6 +18840,9 @@ snapshots: '@esbuild/sunos-x64@0.19.12': optional: true + '@esbuild/sunos-x64@0.21.5': + optional: true + '@esbuild/sunos-x64@0.24.0': optional: true @@ -18537,6 +18864,9 @@ snapshots: '@esbuild/win32-arm64@0.19.12': optional: true + '@esbuild/win32-arm64@0.21.5': + optional: true + '@esbuild/win32-arm64@0.24.0': optional: true @@ -18558,6 +18888,9 @@ snapshots: '@esbuild/win32-ia32@0.19.12': optional: true + '@esbuild/win32-ia32@0.21.5': + optional: true + '@esbuild/win32-ia32@0.24.0': optional: true @@ -18579,6 +18912,9 @@ snapshots: '@esbuild/win32-x64@0.19.12': optional: true + '@esbuild/win32-x64@0.21.5': + optional: true + '@esbuild/win32-x64@0.24.0': optional: true @@ -19150,8 +19486,6 @@ snapshots: '@inquirer/figures@1.0.4': {} - '@ioredis/commands@1.3.0': {} - '@ioredis/commands@1.4.0': {} '@isaacs/cliui@8.0.2': @@ -19177,14 +19511,14 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.7.1 + '@types/node': 20.14.8 jest-mock: 29.7.0 '@jest/fake-timers@29.7.0': dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 24.7.1 + '@types/node': 20.14.8 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -19197,7 +19531,7 @@ snapshots: dependencies: '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/yargs': 15.0.19 chalk: 4.1.2 @@ -19206,13 +19540,13 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/yargs': 17.0.32 chalk: 4.1.2 '@jridgewell/gen-mapping@0.3.13': dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping': 0.3.30 '@jridgewell/gen-mapping@0.3.3': @@ -19224,7 +19558,7 @@ snapshots: '@jridgewell/gen-mapping@0.3.5': dependencies: '@jridgewell/set-array': 1.2.1 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping': 0.3.30 '@jridgewell/remapping@2.3.5': @@ -19252,22 +19586,22 @@ snapshots: '@jridgewell/trace-mapping@0.3.22': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping@0.3.25': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping@0.3.30': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping@0.3.9': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@js-sdsl/ordered-map@4.4.2': {} @@ -23051,18 +23385,18 @@ snapshots: '@slack/logger@3.0.0': dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@slack/logger@4.0.0': dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@slack/oauth@2.6.3': dependencies: '@slack/logger': 3.0.0 '@slack/web-api': 6.12.1 '@types/jsonwebtoken': 8.5.9 - '@types/node': 24.7.1 + '@types/node': 20.14.8 jsonwebtoken: 9.0.2 lodash.isstring: 4.0.1 transitivePeerDependencies: @@ -23073,7 +23407,7 @@ snapshots: '@slack/logger': 4.0.0 '@slack/web-api': 7.5.0 '@types/jsonwebtoken': 9.0.9 - '@types/node': 24.7.1 + '@types/node': 20.14.8 jsonwebtoken: 9.0.2 lodash.isstring: 4.0.1 transitivePeerDependencies: @@ -23083,7 +23417,7 @@ snapshots: dependencies: '@slack/logger': 3.0.0 '@slack/web-api': 6.12.1 - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/ws': 7.4.7 eventemitter3: 5.0.1 finity: 0.5.4 @@ -23100,7 +23434,7 @@ snapshots: '@slack/logger': 3.0.0 '@slack/types': 2.14.0 '@types/is-stream': 1.1.0 - '@types/node': 24.7.1 + '@types/node': 20.14.8 axios: 1.7.7 eventemitter3: 3.1.2 form-data: 2.5.1 @@ -23115,7 +23449,7 @@ snapshots: dependencies: '@slack/logger': 4.0.0 '@slack/types': 2.14.0 - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/retry': 0.12.0 axios: 1.7.7 eventemitter3: 5.0.1 @@ -23741,7 +24075,7 @@ snapshots: '@types/accepts@1.3.7': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/acorn@4.0.6': dependencies: @@ -23775,19 +24109,19 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/bunyan@1.8.9': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/connect@3.4.36': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/connect@3.4.38': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/content-disposition@0.5.8': {} @@ -23798,11 +24132,11 @@ snapshots: '@types/connect': 3.4.38 '@types/express': 4.17.21 '@types/keygrip': 1.0.6 - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/cors@2.8.17': dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/d3-array@3.2.1': {} @@ -23956,14 +24290,14 @@ snapshots: '@types/express-serve-static-core@4.17.43': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/qs': 6.9.11 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 '@types/express-serve-static-core@5.1.0': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/qs': 6.9.11 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -24011,7 +24345,7 @@ snapshots: '@types/is-stream@1.1.0': dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/istanbul-lib-coverage@2.0.6': {} @@ -24027,7 +24361,7 @@ snapshots: '@types/jsonwebtoken@8.5.9': dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/jsonwebtoken@9.0.9': dependencies: @@ -24051,7 +24385,7 @@ snapshots: '@types/http-errors': 2.0.4 '@types/keygrip': 1.0.6 '@types/koa-compose': 3.2.8 - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/koa__router@12.0.3': dependencies: @@ -24085,7 +24419,7 @@ snapshots: '@types/memcached@2.2.10': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/mime@1.3.5': {} @@ -24097,11 +24431,11 @@ snapshots: '@types/mysql@2.15.22': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/mysql@2.15.26': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/nlcst@2.0.3': dependencies: @@ -24111,10 +24445,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@22.18.0': - dependencies: - undici-types: 6.21.0 - '@types/node@24.7.1': dependencies: undici-types: 7.14.0 @@ -24131,14 +24461,10 @@ snapshots: '@types/pg@8.6.1': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 pg-protocol: 1.6.1 pg-types: 2.2.0 - '@types/progress@2.0.7': - dependencies: - '@types/node': 24.7.1 - '@types/promise.allsettled@1.0.6': {} '@types/prop-types@15.7.11': {} @@ -24149,10 +24475,6 @@ snapshots: dependencies: types-ramda: 0.29.7 - '@types/ramda@0.30.1': - dependencies: - types-ramda: 0.30.1 - '@types/ramda@0.30.2': dependencies: types-ramda: 0.30.1 @@ -24161,6 +24483,10 @@ snapshots: dependencies: types-ramda: 0.31.0 + '@types/ramda@0.31.1': + dependencies: + types-ramda: 0.31.0 + '@types/range-parser@1.2.7': {} '@types/react-dom@19.1.8(@types/react@19.1.11)': @@ -24204,13 +24530,13 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/serve-static@1.15.5': dependencies: '@types/http-errors': 2.0.4 '@types/mime': 3.0.4 - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/shimmer@1.2.0': {} @@ -24224,11 +24550,11 @@ snapshots: '@types/tedious@4.0.14': dependencies: - '@types/node': 22.18.0 + '@types/node': 20.14.8 '@types/through@0.0.33': dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/triple-beam@1.3.5': {} @@ -24248,7 +24574,7 @@ snapshots: '@types/ws@7.4.7': dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 '@types/ws@8.5.14': dependencies: @@ -24309,6 +24635,12 @@ snapshots: transitivePeerDependencies: - supports-color + '@vitest/expect@1.6.1': + dependencies: + '@vitest/spy': 1.6.1 + '@vitest/utils': 1.6.1 + chai: 4.5.0 + '@vitest/expect@3.1.3': dependencies: '@vitest/spy': 3.1.3 @@ -24328,21 +24660,44 @@ snapshots: dependencies: tinyrainbow: 2.0.0 + '@vitest/runner@1.6.1': + dependencies: + '@vitest/utils': 1.6.1 + p-limit: 5.0.0 + pathe: 1.1.2 + '@vitest/runner@3.1.3': dependencies: '@vitest/utils': 3.1.3 pathe: 2.0.3 + '@vitest/snapshot@1.6.1': + dependencies: + magic-string: 0.30.19 + pathe: 1.1.2 + pretty-format: 29.7.0 + '@vitest/snapshot@3.1.3': dependencies: '@vitest/pretty-format': 3.1.3 magic-string: 0.30.17 pathe: 2.0.3 + '@vitest/spy@1.6.1': + dependencies: + tinyspy: 2.2.1 + '@vitest/spy@3.1.3': dependencies: tinyspy: 3.0.2 + '@vitest/utils@1.6.1': + dependencies: + diff-sequences: 29.6.3 + estree-walker: 3.0.3 + loupe: 2.3.7 + pretty-format: 29.7.0 + '@vitest/utils@3.1.3': dependencies: '@vitest/pretty-format': 3.1.3 @@ -24513,8 +24868,6 @@ snapshots: ansi-styles@6.2.1: {} - ansis@4.1.0: {} - ansis@4.2.0: {} any-promise@1.3.0: {} @@ -24616,6 +24969,8 @@ snapshots: asap@2.0.6: {} + assertion-error@1.1.0: {} + assertion-error@2.0.1: {} ast-kit@2.1.2: @@ -24693,9 +25048,9 @@ snapshots: xxhash-wasm: 1.1.0 yargs-parser: 21.1.1 yocto-spinner: 0.2.2 - zod: 3.24.2 - zod-to-json-schema: 3.24.5(zod@3.24.2) - zod-to-ts: 1.2.0(typescript@5.9.3)(zod@3.24.2) + zod: 3.25.76 + zod-to-json-schema: 3.24.5(zod@3.25.76) + zod-to-ts: 1.2.0(typescript@5.9.3)(zod@3.25.76) optionalDependencies: sharp: 0.33.5 transitivePeerDependencies: @@ -25078,11 +25433,11 @@ snapshots: bullmq@5.8.7: dependencies: cron-parser: 4.9.0 - ioredis: 5.7.0 + ioredis: 5.8.2 msgpackr: 1.10.1 node-abort-controller: 3.1.1 - semver: 7.6.0 - tslib: 2.6.2 + semver: 7.7.2 + tslib: 2.7.0 uuid: 9.0.1 transitivePeerDependencies: - supports-color @@ -25205,6 +25560,16 @@ snapshots: ccount@2.0.1: {} + chai@4.5.0: + dependencies: + assertion-error: 1.1.0 + check-error: 1.0.3 + deep-eql: 4.1.4 + get-func-name: 2.0.2 + loupe: 2.3.7 + pathval: 1.1.1 + type-detect: 4.1.0 + chai@5.2.0: dependencies: assertion-error: 2.0.1 @@ -25244,6 +25609,10 @@ snapshots: charenc@0.0.2: {} + check-error@1.0.3: + dependencies: + get-func-name: 2.0.2 + check-error@2.1.1: {} cheerio-select@2.1.0: @@ -25305,7 +25674,7 @@ snapshots: chrome-launcher@0.15.2: dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 escape-string-regexp: 4.0.0 is-wsl: 2.2.0 lighthouse-logger: 1.4.2 @@ -25314,7 +25683,7 @@ snapshots: chromium-edge-launcher@1.0.0: dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 escape-string-regexp: 4.0.0 is-wsl: 2.2.0 lighthouse-logger: 1.4.2 @@ -25660,6 +26029,8 @@ snapshots: csstype@3.1.3: {} + csv-parse@6.1.0: {} + d3-array@2.12.1: dependencies: internmap: 1.0.1 @@ -25929,6 +26300,10 @@ snapshots: dependencies: character-entities: 2.0.2 + deep-eql@4.1.4: + dependencies: + type-detect: 4.1.0 + deep-eql@5.0.2: {} deep-extend@0.6.0: {} @@ -26053,6 +26428,8 @@ snapshots: diff-match-patch@1.0.5: {} + diff-sequences@29.6.3: {} + diff@5.2.0: {} diff@8.0.2: {} @@ -26217,7 +26594,7 @@ snapshots: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.17 - '@types/node': 24.7.1 + '@types/node': 20.14.8 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.7.2 @@ -26471,6 +26848,32 @@ snapshots: '@esbuild/win32-ia32': 0.19.12 '@esbuild/win32-x64': 0.19.12 + esbuild@0.21.5: + optionalDependencies: + '@esbuild/aix-ppc64': 0.21.5 + '@esbuild/android-arm': 0.21.5 + '@esbuild/android-arm64': 0.21.5 + '@esbuild/android-x64': 0.21.5 + '@esbuild/darwin-arm64': 0.21.5 + '@esbuild/darwin-x64': 0.21.5 + '@esbuild/freebsd-arm64': 0.21.5 + '@esbuild/freebsd-x64': 0.21.5 + '@esbuild/linux-arm': 0.21.5 + '@esbuild/linux-arm64': 0.21.5 + '@esbuild/linux-ia32': 0.21.5 + '@esbuild/linux-loong64': 0.21.5 + '@esbuild/linux-mips64el': 0.21.5 + '@esbuild/linux-ppc64': 0.21.5 + '@esbuild/linux-riscv64': 0.21.5 + '@esbuild/linux-s390x': 0.21.5 + '@esbuild/linux-x64': 0.21.5 + '@esbuild/netbsd-x64': 0.21.5 + '@esbuild/openbsd-x64': 0.21.5 + '@esbuild/sunos-x64': 0.21.5 + '@esbuild/win32-arm64': 0.21.5 + '@esbuild/win32-ia32': 0.21.5 + '@esbuild/win32-x64': 0.21.5 + esbuild@0.24.0: optionalDependencies: '@esbuild/aix-ppc64': 0.24.0 @@ -27441,6 +27844,8 @@ snapshots: get-east-asian-width@1.3.0: {} + get-func-name@2.0.2: {} + get-intrinsic@1.2.4: dependencies: es-errors: 1.3.0 @@ -28092,20 +28497,6 @@ snapshots: dependencies: loose-envify: 1.4.0 - ioredis@5.7.0: - dependencies: - '@ioredis/commands': 1.3.0 - cluster-key-slot: 1.1.2 - debug: 4.4.0 - denque: 2.1.0 - lodash.defaults: 4.2.0 - lodash.isarguments: 3.1.0 - redis-errors: 1.2.0 - redis-parser: 3.0.0 - standard-as-callback: 2.1.0 - transitivePeerDependencies: - - supports-color - ioredis@5.8.2: dependencies: '@ioredis/commands': 1.4.0 @@ -28388,7 +28779,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.7.1 + '@types/node': 20.14.8 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -28409,13 +28800,13 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 24.7.1 + '@types/node': 20.14.8 jest-util: 29.7.0 jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 24.7.1 + '@types/node': 20.14.8 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -28432,7 +28823,7 @@ snapshots: jest-worker@29.7.0: dependencies: - '@types/node': 24.7.1 + '@types/node': 20.14.8 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -28763,6 +29154,11 @@ snapshots: load-tsconfig@0.2.5: {} + local-pkg@0.5.1: + dependencies: + mlly: 1.8.0 + pkg-types: 1.3.1 + local-pkg@1.1.2: dependencies: mlly: 1.8.0 @@ -28860,6 +29256,10 @@ snapshots: lottie-web@5.12.2: {} + loupe@2.3.7: + dependencies: + get-func-name: 2.0.2 + loupe@3.1.3: {} lowlight@1.20.0: @@ -30282,7 +30682,7 @@ snapshots: dependencies: yocto-queue: 0.1.0 - p-limit@6.1.0: + p-limit@5.0.0: dependencies: yocto-queue: 1.1.1 @@ -30455,6 +30855,8 @@ snapshots: pathe@2.0.3: {} + pathval@1.1.1: {} + pathval@2.0.0: {} peberminta@0.9.0: {} @@ -30757,7 +31159,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 22.18.0 + '@types/node': 20.14.8 long: 5.2.3 proxy-addr@2.0.7: @@ -32462,6 +32864,10 @@ snapshots: strip-json-comments@2.0.1: {} + strip-literal@2.1.1: + dependencies: + js-tokens: 9.0.1 + strip-literal@3.1.0: dependencies: js-tokens: 9.0.1 @@ -32787,10 +33193,14 @@ snapshots: fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 + tinypool@0.8.4: {} + tinypool@1.0.2: {} tinyrainbow@2.0.0: {} + tinyspy@2.2.1: {} + tinyspy@3.0.2: {} tldts-core@6.1.86: {} @@ -32853,7 +33263,7 @@ snapshots: tsdown@0.14.2(typescript@5.9.3): dependencies: - ansis: 4.1.0 + ansis: 4.2.0 cac: 6.7.14 chokidar: 4.0.3 debug: 4.4.1 @@ -32864,7 +33274,7 @@ snapshots: rolldown-plugin-dts: 0.15.9(rolldown@1.0.0-beta.43)(typescript@5.9.3) semver: 7.7.2 tinyexec: 1.0.1 - tinyglobby: 0.2.14 + tinyglobby: 0.2.15 tree-kill: 1.2.2 unconfig: 7.3.3 optionalDependencies: @@ -32915,6 +33325,8 @@ snapshots: type-detect@4.0.8: {} + type-detect@4.1.0: {} + type-fest@0.16.0: {} type-fest@0.21.3: {} @@ -33047,8 +33459,6 @@ snapshots: undici-types@5.26.5: {} - undici-types@6.21.0: {} - undici-types@7.14.0: {} undici@7.14.0: {} @@ -33387,6 +33797,24 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 + vite-node@1.6.1(@types/node@20.14.8)(lightningcss@1.30.1)(terser@5.27.1): + dependencies: + cac: 6.7.14 + debug: 4.4.1 + pathe: 1.1.2 + picocolors: 1.1.1 + vite: 5.4.21(@types/node@20.14.8)(lightningcss@1.30.1)(terser@5.27.1) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vite-node@3.1.3(@types/node@24.7.1)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.27.1)(tsx@4.20.5): dependencies: cac: 6.7.14 @@ -33419,6 +33847,17 @@ snapshots: - supports-color - typescript + vite@5.4.21(@types/node@20.14.8)(lightningcss@1.30.1)(terser@5.27.1): + dependencies: + esbuild: 0.21.5 + postcss: 8.5.6 + rollup: 4.52.5 + optionalDependencies: + '@types/node': 20.14.8 + fsevents: 2.3.3 + lightningcss: 1.30.1 + terser: 5.27.1 + vite@6.3.3(@types/node@24.7.1)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.27.1)(tsx@4.20.5): dependencies: esbuild: 0.25.9 @@ -33475,6 +33914,41 @@ snapshots: optionalDependencies: vite: 6.3.5(@types/node@24.7.1)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.27.1)(tsx@4.20.5) + vitest@1.6.1(@types/node@20.14.8)(jsdom@26.1.0)(lightningcss@1.30.1)(terser@5.27.1): + dependencies: + '@vitest/expect': 1.6.1 + '@vitest/runner': 1.6.1 + '@vitest/snapshot': 1.6.1 + '@vitest/spy': 1.6.1 + '@vitest/utils': 1.6.1 + acorn-walk: 8.3.2 + chai: 4.5.0 + debug: 4.4.1 + execa: 8.0.1 + local-pkg: 0.5.1 + magic-string: 0.30.19 + pathe: 1.1.2 + picocolors: 1.1.1 + std-env: 3.9.0 + strip-literal: 2.1.1 + tinybench: 2.9.0 + tinypool: 0.8.4 + vite: 5.4.21(@types/node@20.14.8)(lightningcss@1.30.1)(terser@5.27.1) + vite-node: 1.6.1(@types/node@20.14.8)(lightningcss@1.30.1)(terser@5.27.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 20.14.8 + jsdom: 26.1.0 + transitivePeerDependencies: + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vitest@3.1.3(@types/debug@4.1.12)(@types/node@24.7.1)(jiti@2.6.1)(jsdom@26.1.0)(lightningcss@1.30.1)(terser@5.27.1)(tsx@4.20.5): dependencies: '@vitest/expect': 3.1.3 @@ -33867,10 +34341,14 @@ snapshots: dependencies: zod: 3.24.2 - zod-to-ts@1.2.0(typescript@5.9.3)(zod@3.24.2): + zod-to-json-schema@3.24.5(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod-to-ts@1.2.0(typescript@5.9.3)(zod@3.25.76): dependencies: typescript: 5.9.3 - zod: 3.24.2 + zod: 3.25.76 zod@3.22.3: {}