first working cli importer

apps/api/src/controllers/import.controller.ts

@@ -1,14 +1,8 @@
 import type { FastifyReply, FastifyRequest } from 'fastify';
-import { pathOr } from 'ramda';
-import { v4 as uuid } from 'uuid';
 
 import { toDots } from '@openpanel/common';
-import type {
-  IClickhouseEvent,
-  IServiceCreateEventPayload,
-} from '@openpanel/db';
-import { ch, formatClickhouseDate } from '@openpanel/db';
-import type { PostEventPayload } from '@openpanel/sdk';
+import type { IClickhouseEvent } from '@openpanel/db';
+import { ch, formatClickhouseDate, TABLE_NAMES } from '@openpanel/db';
 
 export async function importEvents(
   request: FastifyRequest<{
@@ -16,24 +10,31 @@ export async function importEvents(
   }>,
   reply: FastifyReply
 ) {
-  console.log('HERE?!', request.body.length);
-
+  const importedAt = formatClickhouseDate(new Date());
   const values: IClickhouseEvent[] = request.body.map((event) => {
     return {
       ...event,
+      properties: toDots(event.properties),
       project_id: request.client?.projectId ?? '',
       created_at: formatClickhouseDate(event.created_at),
+      imported_at: importedAt,
     };
   });
 
-  const res = await ch.insert({
-    table: 'events',
-    values,
-    format: 'JSONEachRow',
-    clickhouse_settings: {
-      date_time_input_format: 'best_effort',
-    },
-  });
+  try {
+    const res = await ch.insert({
+      table: TABLE_NAMES.events,
+      values,
+      format: 'JSONEachRow',
+      clickhouse_settings: {
+        date_time_input_format: 'best_effort',
+      },
+    });
 
-  reply.send('OK');
+    console.log(res.summary?.written_rows, 'events imported');
+    reply.send('OK');
+  } catch (e) {
+    console.error(e);
+    reply.status(500).send('Error');
+  }
 }

apps/api/src/index.ts

@@ -16,6 +16,7 @@ import { appRouter, createContext } from '@openpanel/trpc';
 
 import eventRouter from './routes/event.router';
 import exportRouter from './routes/export.router';
+import importRouter from './routes/import.router';
 import liveRouter from './routes/live.router';
 import miscRouter from './routes/misc.router';
 import profileRouter from './routes/profile.router';
@@ -91,6 +92,7 @@ const startServer = async () => {
     fastify.register(miscRouter, { prefix: '/misc' });
     fastify.register(exportRouter, { prefix: '/export' });
     fastify.register(webhookRouter, { prefix: '/webhook' });
+    fastify.register(importRouter, { prefix: '/import' });
     fastify.setErrorHandler((error) => {
       logger.error(error, 'Error in request');
     });

apps/dashboard/src/components/overview/useOverviewOptions.ts

@@ -1,6 +1,5 @@
-import { useEffect } from 'react';
+import { differenceInCalendarMonths } from 'date-fns';
 import {
-  parseAsBoolean,
   parseAsInteger,
   parseAsString,
   parseAsStringEnum,
@@ -18,10 +17,6 @@ import { mapKeys } from '@openpanel/validation';
 const nuqsOptions = { history: 'push' } as const;
 
 export function useOverviewOptions() {
-  const [previous, setPrevious] = useQueryState(
-    'compare',
-    parseAsBoolean.withDefault(true).withOptions(nuqsOptions)
-  );
   const [startDate, setStartDate] = useQueryState(
     'start',
     parseAsString.withOptions(nuqsOptions)
@@ -47,8 +42,15 @@ export function useOverviewOptions() {
   );
 
   return {
-    previous,
-    setPrevious,
+    // Skip previous for ranges over 6 months (for performance reasons)
+    previous: !(
+      range === 'yearToDate' ||
+      range === 'lastYear' ||
+      (range === 'custom' &&
+        startDate &&
+        endDate &&
+        differenceInCalendarMonths(startDate, endDate) > 6)
+    ),
     range,
     setRange: (value: IChartRange | null) => {
       if (value !== 'custom') {

apps/dashboard/src/components/report/sidebar/ReportEvents.tsx

@@ -28,9 +28,14 @@ import type { ReportEventMoreProps } from './ReportEventMore';
 export function ReportEvents() {
   const previous = useSelector((state) => state.report.previous);
   const selectedEvents = useSelector((state) => state.report.events);
+  const input = useSelector((state) => state.report);
   const dispatch = useDispatch();
   const { projectId } = useAppParams();
-  const eventNames = useEventNames(projectId);
+  const eventNames = useEventNames(projectId, {
+    startDate: input.startDate,
+    endDate: input.endDate,
+    range: input.range,
+  });
 
   const dispatchChangeEvent = useDebounceFn((event: IChartEvent) => {
     dispatch(changeEvent(event));
@@ -54,7 +59,7 @@ export function ReportEvents() {
       <div className="flex flex-col gap-4">
         {selectedEvents.map((event) => {
           return (
-            <div key={event.id} className="bg-def-100 rounded-lg border">
+            <div key={event.id} className="rounded-lg border bg-def-100">
               <div className="flex items-center gap-2 p-2">
                 <ColorSquare>{event.id}</ColorSquare>
                 <Combobox

apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx

@@ -5,7 +5,7 @@ import { DropdownMenuComposed } from '@/components/ui/dropdown-menu';
 import { RenderDots } from '@/components/ui/RenderDots';
 import { useAppParams } from '@/hooks/useAppParams';
 import { useMappings } from '@/hooks/useMappings';
-import { useDispatch } from '@/redux';
+import { useDispatch, useSelector } from '@/redux';
 import { api } from '@/trpc/client';
 import { SlidersHorizontal, Trash } from 'lucide-react';
 
@@ -26,12 +26,16 @@ interface FilterProps {
 
 export function FilterItem({ filter, event }: FilterProps) {
   const { projectId } = useAppParams();
+  const { range, startDate, endDate } = useSelector((state) => state.report);
   const getLabel = useMappings();
   const dispatch = useDispatch();
   const potentialValues = api.chart.values.useQuery({
     event: event.name,
     property: filter.name,
     projectId,
+    range,
+    startDate,
+    endDate,
   });
 
   const valuesCombobox =
@@ -90,7 +94,7 @@ export function FilterItem({ filter, event }: FilterProps) {
   return (
     <div
       key={filter.name}
-      className="shadow-def-200 px-4 py-2 shadow-[inset_6px_0_0] first:border-t"
+      className="px-4 py-2 shadow-[inset_6px_0_0] shadow-def-200 first:border-t"
     >
       <div className="mb-2 flex items-center gap-2">
         <ColorSquare className="bg-emerald-500">

apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx

@@ -1,6 +1,6 @@
 import { Combobox } from '@/components/ui/combobox';
 import { useAppParams } from '@/hooks/useAppParams';
-import { useDispatch } from '@/redux';
+import { useDispatch, useSelector } from '@/redux';
 import { api } from '@/trpc/client';
 import { FilterIcon } from 'lucide-react';
 
@@ -14,12 +14,16 @@ interface FiltersComboboxProps {
 
 export function FiltersCombobox({ event }: FiltersComboboxProps) {
   const dispatch = useDispatch();
+  const { range, startDate, endDate } = useSelector((state) => state.report);
   const { projectId } = useAppParams();
 
   const query = api.chart.properties.useQuery(
     {
       event: event.name,
       projectId,
+      range,
+      startDate,
+      endDate,
     },
     {
       enabled: !!event.name,

apps/dashboard/src/hooks/useEventNames.ts

@@ -1,8 +1,9 @@
 import { api } from '@/trpc/client';
 
-export function useEventNames(projectId: string) {
+export function useEventNames(projectId: string, options?: any) {
   const query = api.chart.events.useQuery({
     projectId: projectId,
+    ...(options ? options : {}),
   });
 
   return query.data ?? [];

apps/worker/src/jobs/events.incoming-event.ts

@@ -1,11 +1,15 @@
 import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';
 import { parseUserAgent } from '@/utils/parse-user-agent';
-import { isSameDomain, parsePath } from '@/utils/url';
 import type { Job } from 'bullmq';
 import { omit } from 'ramda';
 import { v4 as uuid } from 'uuid';
 
-import { getTime, toISOString } from '@openpanel/common';
+import {
+  getTime,
+  isSameDomain,
+  parsePath,
+  toISOString,
+} from '@openpanel/common';
 import type { IServiceCreateEventPayload } from '@openpanel/db';
 import { createEvent } from '@openpanel/db';
 import { getLastScreenViewFromProfileId } from '@openpanel/db/src/services/event.service';
@@ -97,6 +101,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
       referrerType: event?.referrerType ?? '',
       profile: undefined,
       meta: undefined,
+      importedAt: null,
     };
 
     return createEvent(payload);
@@ -170,8 +175,6 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
     referrer: referrer?.url,
     referrerName: referrer?.name || utmReferrer?.name || '',
     referrerType: referrer?.type || utmReferrer?.type || '',
-    profile: undefined,
-    meta: undefined,
   };
 
   if (!sessionEnd) {

apps/worker/src/utils/url.ts

@@ -1,52 +0,0 @@
-export function parseSearchParams(
-  params: URLSearchParams
-): Record<string, string> | undefined {
-  const result: Record<string, string> = {};
-  for (const [key, value] of params.entries()) {
-    result[key] = value;
-  }
-  return Object.keys(result).length ? result : undefined;
-}
-
-export function parsePath(path?: string): {
-  query?: Record<string, string>;
-  path: string;
-  origin: string;
-  hash?: string;
-} {
-  if (!path) {
-    return {
-      path: '',
-      origin: '',
-    };
-  }
-
-  try {
-    const url = new URL(path);
-    return {
-      query: parseSearchParams(url.searchParams),
-      path: url.pathname,
-      hash: url.hash || undefined,
-      origin: url.origin,
-    };
-  } catch (error) {
-    return {
-      path,
-      origin: '',
-    };
-  }
-}
-
-export function isSameDomain(
-  url1: string | undefined,
-  url2: string | undefined
-) {
-  if (!url1 || !url2) {
-    return false;
-  }
-  try {
-    return new URL(url1).hostname === new URL(url2).hostname;
-  } catch (e) {
-    return false;
-  }
-}