wip event list

packages/db/src/clickhouse-client.ts

@@ -14,7 +14,7 @@ interface ClickhouseJsonResponse<T> {
   meta: { name: string; type: string }[];
 }
 
-export async function chQueryAll<T extends Record<string, any>>(
+export async function chQueryWithMeta<T extends Record<string, any>>(
   query: string
 ): Promise<ClickhouseJsonResponse<T>> {
   const res = await ch.query({
@@ -42,7 +42,7 @@ export async function chQueryAll<T extends Record<string, any>>(
 export async function chQuery<T extends Record<string, any>>(
   query: string
 ): Promise<T[]> {
-  return (await chQueryAll<T>(query)).data;
+  return (await chQueryWithMeta<T>(query)).data;
 }
 
 export function formatClickhouseDate(_date: Date | string) {

packages/db/src/services/chart.service.ts

@@ -0,0 +1,178 @@
+import { formatClickhouseDate } from '../clickhouse-client';
+import type { SqlBuilderObject } from '../sql-builder';
+import { createSqlBuilder } from '../sql-builder';
+
+function log(sql: string) {
+  const logs = ['--- START', sql, '--- END'];
+  console.log(logs.join('\n'));
+  return sql;
+}
+
+type IGetChartDataInput = any;
+
+export function getChartSql({
+  event,
+  breakdowns,
+  interval,
+  startDate,
+  endDate,
+  projectId,
+}: IGetChartDataInput) {
+  const { sb, join, getWhere, getFrom, getSelect, getOrderBy, getGroupBy } =
+    createSqlBuilder();
+
+  sb.where.projectId = `project_id = '${projectId}'`;
+  if (event.name !== '*') {
+    sb.select.label = `'${event.name}' as label`;
+    sb.where.eventName = `name = '${event.name}'`;
+  }
+
+  getEventFiltersWhereClause(sb, event.filters);
+
+  sb.select.count = `count(*) as count`;
+  switch (interval) {
+    case 'minute': {
+      sb.select.date = `toStartOfMinute(created_at) as date`;
+      break;
+    }
+    case 'hour': {
+      sb.select.date = `toStartOfHour(created_at) as date`;
+      break;
+    }
+    case 'day': {
+      sb.select.date = `toStartOfDay(created_at) as date`;
+      break;
+    }
+    case 'month': {
+      sb.select.date = `toStartOfMonth(created_at) as date`;
+      break;
+    }
+  }
+  sb.groupBy.date = 'date';
+  sb.orderBy.date = 'date ASC';
+
+  if (startDate) {
+    sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
+  }
+
+  if (endDate) {
+    sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;
+  }
+
+  const breakdown = breakdowns[0]!;
+  if (breakdown) {
+    const value = breakdown.name.startsWith('properties.')
+      ? `mapValues(mapExtractKeyLike(properties, '${breakdown.name
+          .replace(/^properties\./, '')
+          .replace('.*.', '.%.')}'))`
+      : breakdown.name;
+    sb.select.label = breakdown.name.startsWith('properties.')
+      ? `arrayElement(${value}, 1) as label`
+      : `${breakdown.name} as label`;
+    sb.groupBy.label = `label`;
+  }
+
+  if (event.segment === 'user') {
+    sb.select.count = `countDistinct(profile_id) as count`;
+  }
+
+  if (event.segment === 'user_average') {
+    sb.select.count = `COUNT(*)::float / COUNT(DISTINCT profile_id)::float as count`;
+  }
+
+  if (event.segment === 'property_sum' && event.property) {
+    sb.select.count = `sum(${event.property}) as count`;
+  }
+
+  if (event.segment === 'property_average' && event.property) {
+    sb.select.count = `avg(${event.property}) as count`;
+  }
+
+  if (event.segment === 'one_event_per_user') {
+    sb.from = `(
+      SELECT DISTINCT ON (profile_id) * from events WHERE ${join(
+        sb.where,
+        ' AND '
+      )}
+        ORDER BY profile_id, created_at DESC
+      ) as subQuery`;
+
+    return log(`${getSelect()} ${getFrom()} ${getGroupBy()} ${getOrderBy()}`);
+  }
+
+  return log(
+    `${getSelect()} ${getFrom()} ${getWhere()} ${getGroupBy()} ${getOrderBy()}`
+  );
+}
+
+export function getEventFiltersWhereClause(
+  sb: SqlBuilderObject,
+  filters: any[]
+) {
+  filters.forEach((filter, index) => {
+    const id = `f${index}`;
+    const { name, value, operator } = filter;
+
+    if (name.startsWith('properties.')) {
+      const whereFrom = `mapValues(mapExtractKeyLike(properties, '${name
+        .replace(/^properties\./, '')
+        .replace('.*.', '.%.')}'))`;
+
+      switch (operator) {
+        case 'is': {
+          sb.where[id] = `arrayExists(x -> ${value
+            .map((val) => `x = '${String(val).trim()}'`)
+            .join(' OR ')}, ${whereFrom})`;
+          break;
+        }
+        case 'isNot': {
+          sb.where[id] = `arrayExists(x -> ${value
+            .map((val) => `x != '${String(val).trim()}'`)
+            .join(' OR ')}, ${whereFrom})`;
+          break;
+        }
+        case 'contains': {
+          sb.where[id] = `arrayExists(x -> ${value
+            .map((val) => `x LIKE '%${String(val).trim()}%'`)
+            .join(' OR ')}, ${whereFrom})`;
+          break;
+        }
+        case 'doesNotContain': {
+          sb.where[id] = `arrayExists(x -> ${value
+            .map((val) => `x NOT LIKE '%${String(val).trim()}%'`)
+            .join(' OR ')}, ${whereFrom})`;
+          break;
+        }
+      }
+    } else {
+      switch (operator) {
+        case 'is': {
+          sb.where[id] = `${name} IN (${value
+            .map((val) => `'${String(val).trim()}'`)
+            .join(', ')})`;
+          break;
+        }
+        case 'isNot': {
+          sb.where[id] = `${name} NOT IN (${value
+            .map((val) => `'${String(val).trim()}'`)
+            .join(', ')})`;
+          break;
+        }
+        case 'contains': {
+          sb.where[id] = value
+            .map((val) => `${name} LIKE '%${String(val).trim()}%'`)
+            .join(' OR ');
+          break;
+        }
+        case 'doesNotContain': {
+          sb.where[id] = value
+            .map((val) => `${name} NOT LIKE '%${String(val).trim()}%'`)
+            .join(' OR ');
+          break;
+        }
+      }
+    }
+  });
+
+  return sb;
+}

packages/db/src/services/event.service.ts

@@ -1,4 +1,4 @@
-import { omit } from 'ramda';
+import { omit, uniq } from 'ramda';
 import { v4 as uuid } from 'uuid';
 
 import { randomSplitName, toDots } from '@mixan/common';
@@ -10,10 +10,11 @@ import {
   convertClickhouseDateToJs,
   formatClickhouseDate,
 } from '../clickhouse-client';
-import type { Prisma } from '../prisma-client';
+import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 import type { IDBProfile } from '../prisma-types';
 import { createSqlBuilder } from '../sql-builder';
+import { getEventFiltersWhereClause } from './chart.service';
 
 export interface IClickhouseEvent {
   id: string;
@@ -37,7 +38,10 @@ export interface IClickhouseEvent {
   device: string;
   brand: string;
   model: string;
+
+  // They do not exist here. Just make ts happy for now
   profile?: IDBProfile;
+  meta?: EventMeta;
 }
 
 export function transformEvent(
@@ -66,6 +70,7 @@ export function transformEvent(
     referrerName: event.referrer_name,
     referrerType: event.referrer_type,
     profile: event.profile,
+    meta: event.meta,
   };
 }
 
@@ -95,11 +100,13 @@ export interface IServiceCreateEventPayload {
   referrer: string | undefined;
   referrerName: string | undefined;
   referrerType: string | undefined;
-  profile?: IDBProfile;
+  profile: IDBProfile | undefined;
+  meta: EventMeta | undefined;
 }
 
 interface GetEventsOptions {
   profile?: boolean | Prisma.ProfileSelect;
+  meta?: boolean | Prisma.EventMetaSelect;
 }
 
 export async function getLiveVisitors(projectId: string) {
@@ -129,6 +136,22 @@ export async function getEvents(
         | undefined;
     }
   }
+
+  if (options.meta) {
+    const names = uniq(events.map((e) => e.name));
+    const metas = await db.eventMeta.findMany({
+      where: {
+        name: {
+          in: names,
+        },
+        project_id: events[0]?.project_id,
+      },
+      select: options.meta === true ? undefined : options.meta,
+    });
+    for (const event of events) {
+      event.meta = metas.find((m) => m.name === event.name);
+    }
+  }
   return events.map(transformEvent);
 }
 
@@ -227,7 +250,8 @@ interface GetEventListOptions {
   projectId: string;
   profileId?: string;
   take: number;
-  cursor?: string;
+  cursor?: number;
+  filters: any[];
 }
 
 export async function getEventList({
@@ -235,22 +259,44 @@ export async function getEventList({
   take,
   projectId,
   profileId,
+  filters,
 }: GetEventListOptions) {
   const { sb, getSql } = createSqlBuilder();
 
   sb.limit = take;
+  sb.offset = (cursor ?? 0) * take;
   sb.where.projectId = `project_id = '${projectId}'`;
   if (profileId) {
     sb.where.profileId = `profile_id = '${profileId}'`;
   }
 
-  if (cursor) {
-    sb.where.cursor = `created_at <= '${formatClickhouseDate(cursor)}'`;
-  }
+  getEventFiltersWhereClause(sb, filters);
+
+  // if (cursor) {
+  //   sb.where.cursor = `created_at <= '${formatClickhouseDate(cursor)}'`;
+  // }
 
   sb.orderBy.created_at = 'created_at DESC';
 
-  const res = await getEvents(getSql(), { profile: true });
-
-  return res;
+  return getEvents(getSql(), { profile: true, meta: true });
+}
+
+export async function getEventsCount({
+  projectId,
+  profileId,
+  filters,
+}: Omit<GetEventListOptions, 'cursor' | 'take'>) {
+  const { sb, getSql } = createSqlBuilder();
+  sb.where.projectId = `project_id = '${projectId}'`;
+  if (profileId) {
+    sb.where.profileId = `profile_id = '${profileId}'`;
+  }
+
+  getEventFiltersWhereClause(sb, filters);
+
+  const res = await chQuery<{ count: number }>(
+    getSql().replace('*', 'count(*) as count')
+  );
+
+  return res[0]?.count ?? 0;
 }

packages/db/src/sql-builder.ts

@@ -1,16 +1,18 @@
+export interface SqlBuilderObject {
+  where: Record<string, string>;
+  select: Record<string, string>;
+  groupBy: Record<string, string>;
+  orderBy: Record<string, string>;
+  from: string;
+  limit: number | undefined;
+  offset: number | undefined;
+}
+
 export function createSqlBuilder() {
   const join = (obj: Record<string, string> | string[], joiner: string) =>
     Object.values(obj).filter(Boolean).join(joiner);
 
-  const sb: {
-    where: Record<string, string>;
-    select: Record<string, string>;
-    groupBy: Record<string, string>;
-    orderBy: Record<string, string>;
-    from: string;
-    limit: number | undefined;
-    offset: number | undefined;
-  } = {
+  const sb: SqlBuilderObject = {
     where: {},
     from: 'openpanel.events',
     select: {},