feat(dashboard): reuse reports filter on overview and add more operators (#31)

packages/constants/index.ts

@@ -67,6 +67,9 @@ export const operators = {
   isNot: 'Is not',
   contains: 'Contains',
   doesNotContain: 'Not contains',
+  startsWith: 'Starts with',
+  endsWith: 'Ends with',
+  regex: 'Regex',
 } as const;
 
 export const chartTypes = {

packages/db/src/clickhouse-client.ts

@@ -43,7 +43,8 @@ export const ch = new Proxy(originalCh, {
         } catch (error: unknown) {
           if (
             error instanceof Error &&
-            (error.message.includes('socket hang up') ||
+            (error.message.includes('Connect') ||
+              error.message.includes('socket hang up') ||
               error.message.includes('Timeout error'))
           ) {
             console.info(
@@ -65,8 +66,7 @@ export const ch = new Proxy(originalCh, {
             }
           } else {
             if (args[0].query) {
-              console.log('FAILED QUERY:');
-              console.log(args[0].query);
+              console.log('FAILED QUERY:', args[0].query);
             }
 
             // Handle other errors or rethrow them

packages/db/src/services/chart.service.ts

@@ -13,11 +13,26 @@ import {
 } from '../clickhouse-client';
 import { createSqlBuilder } from '../sql-builder';
 
-function getPropertyKey(property: string) {
+export function transformPropertyKey(property: string) {
+  if (property.startsWith('properties.')) {
+    if (property.includes('*')) {
+      return property
+        .replace(/^properties\./, '')
+        .replace('.*.', '.%.')
+        .replace(/\[\*\]$/, '.%')
+        .replace(/\[\*\].?/, '.%.');
+    }
+    return `properties['${property.replace(/^properties\./, '')}']`;
+  }
+
+  return property;
+}
+
+export function getSelectPropertyKey(property: string) {
   if (property.startsWith('properties.')) {
     if (property.includes('*')) {
       return `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
-        property.replace(/^properties\./, '').replace('.*.', '.%.')
+        transformPropertyKey(property)
       )})))`;
     }
     return `properties['${property.replace(/^properties\./, '')}']`;
@@ -79,11 +94,11 @@ export function getChartSql({
   }
 
   if (breakdowns.length > 0 && limit) {
-    sb.where.bar = `(${breakdowns.map((b) => getPropertyKey(b.name)).join(',')}) IN (
-      SELECT ${breakdowns.map((b) => getPropertyKey(b.name)).join(',')}
+    sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (
+      SELECT ${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}
       FROM ${TABLE_NAMES.events}
       ${getWhere()}
-      GROUP BY ${breakdowns.map((b) => getPropertyKey(b.name)).join(',')}
+      GROUP BY ${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}
       ORDER BY count(*) DESC
       LIMIT ${limit}
     )`;
@@ -91,7 +106,7 @@ export function getChartSql({
 
   breakdowns.forEach((breakdown, index) => {
     const key = `label_${index}`;
-    sb.select[key] = `${getPropertyKey(breakdown.name)} as ${key}`;
+    sb.select[key] = `${getSelectPropertyKey(breakdown.name)} as ${key}`;
     sb.groupBy[key] = `${key}`;
   });
 
@@ -108,13 +123,13 @@ export function getChartSql({
   }
 
   if (event.segment === 'property_sum' && event.property) {
-    sb.select.count = `sum(toFloat64(${getPropertyKey(event.property)})) as count`;
-    sb.where.property = `${getPropertyKey(event.property)} IS NOT NULL`;
+    sb.select.count = `sum(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
+    sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL`;
   }
 
   if (event.segment === 'property_average' && event.property) {
-    sb.select.count = `avg(toFloat64(${getPropertyKey(event.property)})) as count`;
-    sb.where.property = `${getPropertyKey(event.property)} IS NOT NULL`;
+    sb.select.count = `avg(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
+    sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL`;
   }
 
   if (event.segment === 'one_event_per_user') {
@@ -150,11 +165,9 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
     }
 
     if (name.startsWith('properties.')) {
-      const propertyKey = name
-        .replace(/^properties\./, '')
-        .replace('.*.', '.%.');
+      const propertyKey = getSelectPropertyKey(name);
       const isWildcard = propertyKey.includes('%');
-      const whereFrom = getPropertyKey(name);
+      const whereFrom = getSelectPropertyKey(name);
 
       switch (operator) {
         case 'is': {
@@ -211,6 +224,48 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
           }
           break;
         }
+        case 'startsWith': {
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> ${value
+              .map((val) => `x LIKE ${escape(`${String(val).trim()}%`)}`)
+              .join(' OR ')}, ${whereFrom})`;
+          } else {
+            where[id] = value
+              .map(
+                (val) => `${whereFrom} LIKE ${escape(`${String(val).trim()}%`)}`
+              )
+              .join(' OR ');
+          }
+          break;
+        }
+        case 'endsWith': {
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> ${value
+              .map((val) => `x LIKE ${escape(`%${String(val).trim()}`)}`)
+              .join(' OR ')}, ${whereFrom})`;
+          } else {
+            where[id] = value
+              .map(
+                (val) => `${whereFrom} LIKE ${escape(`%${String(val).trim()}`)}`
+              )
+              .join(' OR ');
+          }
+          break;
+        }
+        case 'regex': {
+          if (isWildcard) {
+            where[id] = `arrayExists(x -> ${value
+              .map((val) => `match(x, ${escape(String(val).trim())})`)
+              .join(' OR ')}, ${whereFrom})`;
+          } else {
+            where[id] = value
+              .map(
+                (val) => `match(${whereFrom}, ${escape(String(val).trim())})`
+              )
+              .join(' OR ');
+          }
+          break;
+        }
       }
     } else {
       switch (operator) {
@@ -240,6 +295,24 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
             .join(' OR ');
           break;
         }
+        case 'startsWith': {
+          where[id] = value
+            .map((val) => `${name} LIKE ${escape(`${String(val).trim()}%`)}`)
+            .join(' OR ');
+          break;
+        }
+        case 'endsWith': {
+          where[id] = value
+            .map((val) => `${name} LIKE ${escape(`%${String(val).trim()}`)}`)
+            .join(' OR ');
+          break;
+        }
+        case 'regex': {
+          where[id] = value
+            .map((val) => `match(${name}, ${escape(String(val).trim())})`)
+            .join(' OR ');
+          break;
+        }
       }
     }
   });

packages/trpc/src/routers/chart.ts

@@ -7,6 +7,8 @@ import {
   createSqlBuilder,
   db,
   formatClickhouseDate,
+  getPropertyKey,
+  getSelectPropertyKey,
   TABLE_NAMES,
   toDate,
 } from '@openpanel/db';
@@ -126,14 +128,7 @@ export const chartRouter = createTRPCRouter({
       if (event !== '*') {
         sb.where.event = `name = ${escape(event)}`;
       }
-      if (property.startsWith('properties.')) {
-        sb.select.values = `distinct arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
-          property.replace(/^properties\./, '').replace('.*.', '.%.')
-        )}))) as values`;
-      } else {
-        sb.select.values = `distinct ${property} as values`;
-      }
-
+      sb.select.values = `distinct ${getSelectPropertyKey(property)} as values`;
       sb.where.date = `${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`;
 
       const events = await chQuery<{ values: string[] }>(getSql());