first working cli importer

packages/db/src/clickhouse-client.ts

@@ -11,8 +11,8 @@ export const originalCh = createClient({
   username: process.env.CLICKHOUSE_USER,
   password: process.env.CLICKHOUSE_PASSWORD,
   database: process.env.CLICKHOUSE_DB,
-  max_open_connections: 10,
-  request_timeout: 10000,
+  max_open_connections: 30,
+  request_timeout: 30000,
   keep_alive: {
     enabled: true,
     idle_socket_ttl: 8000,
@@ -92,7 +92,7 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
   };
 
   console.log(
-    `Query: (${Date.now() - start}ms, ${response.statistics?.elapsed}ms)`,
+    `Query: (${Date.now() - start}ms, ${response.statistics?.elapsed}ms), Rows: ${json.rows}`,
     query
   );
 

packages/db/src/services/chart.service.ts

@@ -125,33 +125,67 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
     }
 
     if (name.startsWith('properties.')) {
+      const propertyKey = name
+        .replace(/^properties\./, '')
+        .replace('.*.', '.%.');
+      const isWildcard = propertyKey.includes('%');
       const whereFrom = `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
         name.replace(/^properties\./, '').replace('.*.', '.%.')
       )})))`;
 
       switch (operator) {
         case 'is': {
-          where[id] = `arrayExists(x -> ${value
-            .map((val) => `x = ${escape(String(val).trim())}`)
-            .join(' OR ')}, ${whereFrom})`;
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> ${value
+              .map((val) => `x = ${escape(String(val).trim())}`)
+              .join(' OR ')}, ${whereFrom})`;
+          } else {
+            where[id] = `properties['${propertyKey}'] IN (${value
+              .map((val) => escape(String(val).trim()))
+              .join(', ')})`;
+          }
           break;
         }
         case 'isNot': {
-          where[id] = `arrayExists(x -> ${value
-            .map((val) => `x != ${escape(String(val).trim())}`)
-            .join(' OR ')}, ${whereFrom})`;
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> ${value
+              .map((val) => `x != ${escape(String(val).trim())}`)
+              .join(' OR ')}, ${whereFrom})`;
+          } else {
+            where[id] = `properties['${propertyKey}'] NOT IN (${value
+              .map((val) => escape(String(val).trim()))
+              .join(', ')})`;
+          }
           break;
         }
         case 'contains': {
-          where[id] = `arrayExists(x -> ${value
-            .map((val) => `x LIKE ${escape(`%${String(val).trim()}%`)}`)
-            .join(' OR ')}, ${whereFrom})`;
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> ${value
+              .map((val) => `x LIKE ${escape(`%${String(val).trim()}%`)}`)
+              .join(' OR ')}, ${whereFrom})`;
+          } else {
+            where[id] = value
+              .map(
+                (val) =>
+                  `properties['${propertyKey}'] LIKE ${escape(`%${String(val).trim()}%`)}`
+              )
+              .join(' OR ');
+          }
           break;
         }
         case 'doesNotContain': {
-          where[id] = `arrayExists(x -> ${value
-            .map((val) => `x NOT LIKE ${escape(`%${String(val).trim()}%`)}`)
-            .join(' OR ')}, ${whereFrom})`;
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> ${value
+              .map((val) => `x NOT LIKE ${escape(`%${String(val).trim()}%`)}`)
+              .join(' OR ')}, ${whereFrom})`;
+          } else {
+            where[id] = value
+              .map(
+                (val) =>
+                  `properties['${propertyKey}'] NOT LIKE ${escape(`%${String(val).trim()}%`)}`
+              )
+              .join(' OR ');
+          }
           break;
         }
       }

packages/db/src/services/event.service.ts

@@ -21,6 +21,13 @@ import { getEventFiltersWhereClause } from './chart.service';
 import { getProfiles, upsertProfile } from './profile.service';
 import type { IServiceProfile } from './profile.service';
 
+export type IImportedEvent = Omit<
+  IClickhouseEvent,
+  'properties' | 'profile' | 'meta' | 'imported_at'
+> & {
+  properties: Record<string, unknown>;
+};
+
 export interface IClickhouseEvent {
   id: string;
   name: string;
@@ -34,7 +41,7 @@ export interface IClickhouseEvent {
   referrer_name: string;
   referrer_type: string;
   duration: number;
-  properties: Record<string, string | number | boolean>;
+  properties: Record<string, string | number | boolean | undefined | null>;
   created_at: string;
   country: string;
   city: string;
@@ -48,6 +55,7 @@ export interface IClickhouseEvent {
   device: string;
   brand: string;
   model: string;
+  imported_at: string | null;
 
   // They do not exist here. Just make ts happy for now
   profile?: IServiceProfile;
@@ -86,6 +94,7 @@ export function transformEvent(
     referrerType: event.referrer_type,
     profile: event.profile,
     meta: event.meta,
+    importedAt: event.imported_at ? new Date(event.imported_at) : null,
   };
 }
 
@@ -119,6 +128,7 @@ export interface IServiceCreateEventPayload {
   referrer: string | undefined;
   referrerName: string | undefined;
   referrerType: string | undefined;
+  importedAt: Date | null;
   profile: IServiceProfile | undefined;
   meta: EventMeta | undefined;
 }
@@ -221,7 +231,10 @@ export async function getEvents(
 }
 
 export async function createEvent(
-  payload: Omit<IServiceCreateEventPayload, 'id'>
+  payload: Omit<
+    IServiceCreateEventPayload,
+    'id' | 'importedAt' | 'profile' | 'meta'
+  >
 ) {
   if (!payload.profileId) {
     payload.profileId = payload.deviceId;
@@ -283,6 +296,7 @@ export async function createEvent(
     referrer: payload.referrer ?? '',
     referrer_name: payload.referrerName ?? '',
     referrer_type: payload.referrerType ?? '',
+    imported_at: null,
   };
 
   await eventBuffer.insert(event);

packages/db/src/services/profile.service.ts

@@ -64,17 +64,16 @@ interface GetProfileListOptions {
 }
 
 export async function getProfiles(ids: string[]) {
-  if (ids.length === 0) {
+  const filteredIds = ids.filter((id) => id !== '');
+
+  if (filteredIds.length === 0) {
     return [];
   }
 
   const data = await chQuery<IClickhouseProfile>(
     `SELECT *
     FROM profiles FINAL 
-    WHERE id IN (${ids
-      .map((id) => escape(id))
-      .filter(Boolean)
-      .join(',')})
+    WHERE id IN (${filteredIds.map((id) => escape(id)).join(',')})
     `
   );