fix: timezone issue + improvements for funnel and conversion charts

packages/db/src/clickhouse/query-builder.ts

@@ -508,7 +508,10 @@ export class Query<T = any> {
   // Execution methods
   async execute(): Promise<T[]> {
     const query = this.buildQuery();
-    console.log('query', query);
+    console.log(
+      'query',
+      `${query} SETTINGS session_timezone = '${this.timezone}'`,
+    );
 
     const result = await this.client.query({
       query,

packages/db/src/services/conversion.service.ts

@@ -20,7 +20,10 @@ export class ConversionService {
     events,
     breakdowns = [],
     interval,
-  }: Omit<IChartInput, 'range' | 'previous' | 'metric' | 'chartType'>) {
+    timezone,
+  }: Omit<IChartInput, 'range' | 'previous' | 'metric' | 'chartType'> & {
+    timezone: string;
+  }) {
     const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';
     const breakdownColumns = breakdowns.map(
       (b, index) => `${getSelectPropertyKey(b.name)} as b_${index}`,
@@ -44,7 +47,7 @@ export class ConversionService {
       getEventFiltersWhereClause(eventB.filters),
     ).join(' AND ');
 
-    const eventACte = clix(this.client)
+    const eventACte = clix(this.client, timezone)
       .select([
         `DISTINCT ${group}`,
         'created_at AS a_time',
@@ -56,22 +59,22 @@ export class ConversionService {
       .where('name', '=', eventA.name)
       .rawWhere(whereA)
       .where('created_at', 'BETWEEN', [
-        clix.datetime(startDate),
-        clix.datetime(endDate),
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
       ]);
 
-    const eventBCte = clix(this.client)
+    const eventBCte = clix(this.client, timezone)
       .select([group, 'created_at AS b_time'])
       .from(TABLE_NAMES.events)
       .where('project_id', '=', projectId)
       .where('name', '=', eventB.name)
       .rawWhere(whereB)
       .where('created_at', 'BETWEEN', [
-        clix.datetime(startDate),
-        clix.datetime(endDate),
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
       ]);
 
-    const query = clix(this.client)
+    const query = clix(this.client, timezone)
       .with('event_a', eventACte)
       .with('event_b', eventBCte)
       .select<{

packages/db/src/services/funnel.service.ts

@@ -1,6 +1,6 @@
 import { ifNaN } from '@openpanel/common';
 import type { IChartEvent, IChartInput } from '@openpanel/validation';
-import { last, reverse } from 'ramda';
+import { last, reverse, uniq } from 'ramda';
 import sqlstring from 'sqlstring';
 import { ch } from '../clickhouse/client';
 import { TABLE_NAMES } from '../clickhouse/client';
@@ -98,6 +98,14 @@ export class FunnelService {
     return Object.values(series);
   }
 
+  getProfileFilters(events: IChartEvent[]) {
+    return events.flatMap((e) =>
+      e.filters
+        ?.filter((f) => f.name.startsWith('profile.'))
+        .map((f) => f.name.replace('profile.', '')),
+    );
+  }
+
   async getFunnel({
     projectId,
     startDate,
@@ -106,7 +114,8 @@ export class FunnelService {
     funnelWindow = 24,
     funnelGroup,
     breakdowns = [],
-  }: IChartInput) {
+    timezone = 'UTC',
+  }: IChartInput & { timezone: string }) {
     if (!startDate || !endDate) {
       throw new Error('startDate and endDate are required');
     }
@@ -118,9 +127,14 @@ export class FunnelService {
     const funnelWindowSeconds = funnelWindow * 3600;
     const group = this.getFunnelGroup(funnelGroup);
     const funnels = this.getFunnelConditions(events);
+    const profileFilters = this.getProfileFilters(events);
+    const anyFilterOnProfile = profileFilters.length > 0;
+    const anyBreakdownOnProfile = breakdowns.some((b) =>
+      b.name.startsWith('profile.'),
+    );
 
     // Create the funnel CTE
-    const funnelCte = clix(this.client)
+    const funnelCte = clix(this.client, timezone)
       .select([
         `${group[0]} AS ${group[1]}`,
         ...breakdowns.map(
@@ -131,8 +145,8 @@ export class FunnelService {
       .from(TABLE_NAMES.events, false)
       .where('project_id', '=', projectId)
       .where('created_at', 'BETWEEN', [
-        clix.datetime(startDate),
-        clix.datetime(endDate),
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
       ])
       .where(
         'name',
@@ -141,21 +155,29 @@ export class FunnelService {
       )
       .groupBy([group[1], ...breakdowns.map((b, index) => `b_${index}`)]);
 
+    if (anyFilterOnProfile || anyBreakdownOnProfile) {
+      funnelCte.leftJoin(
+        `(SELECT id, ${uniq(profileFilters.map((f) => f.split('.')[0]))} FROM ${TABLE_NAMES.profiles} FINAL
+          WHERE project_id = ${sqlstring.escape(projectId)}) as profile`,
+        'profile.id = profile_id',
+      );
+    }
+
     // Create the sessions CTE if needed
     const sessionsCte =
       group[0] !== 'session_id'
-        ? clix(this.client)
+        ? clix(this.client, timezone)
             .select(['profile_id', 'id'])
             .from(TABLE_NAMES.sessions)
             .where('project_id', '=', projectId)
             .where('created_at', 'BETWEEN', [
-              clix.datetime(startDate),
-              clix.datetime(endDate),
+              clix.datetime(startDate, 'toDateTime'),
+              clix.datetime(endDate, 'toDateTime'),
             ])
         : null;
 
     // Base funnel query with CTEs
-    const funnelQuery = clix(this.client);
+    const funnelQuery = clix(this.client, timezone);
 
     if (sessionsCte) {
       funnelCte.leftJoin('sessions s', 's.id = session_id');
@@ -202,7 +224,7 @@ export class FunnelService {
                 {
                   event: {
                     ...event,
-                    displayName: event.displayName ?? event.name,
+                    displayName: event.displayName || event.name,
                   },
                   count: item.count,
                   percent: (item.count / totalSessions) * 100,

packages/db/src/services/profile.service.ts

@@ -10,13 +10,14 @@ import {
   TABLE_NAMES,
   ch,
   chQuery,
+  convertClickhouseDateToJs,
   formatClickhouseDate,
 } from '../clickhouse/client';
 import { createSqlBuilder } from '../sql-builder';
 
 export type IProfileMetrics = {
-  lastSeen: string;
-  firstSeen: string;
+  lastSeen: Date;
+  firstSeen: Date;
   screenViews: number;
   sessions: number;
   durationAvg: number;
@@ -29,7 +30,12 @@ export type IProfileMetrics = {
   avgTimeBetweenSessions: number;
 };
 export function getProfileMetrics(profileId: string, projectId: string) {
-  return chQuery<IProfileMetrics>(`
+  return chQuery<
+    Omit<IProfileMetrics, 'lastSeen' | 'firstSeen'> & {
+      lastSeen: string;
+      firstSeen: string;
+    }
+  >(`
     WITH lastSeen AS (
       SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
     ),
@@ -84,7 +90,15 @@ export function getProfileMetrics(profileId: string, projectId: string) {
       (SELECT avgEventsPerSession FROM avgEventsPerSession) as avgEventsPerSession,
       (SELECT conversionEvents FROM conversionEvents) as conversionEvents,
       (SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions
-  `).then((data) => data[0]!);
+  `)
+    .then((data) => data[0]!)
+    .then((data) => {
+      return {
+        ...data,
+        lastSeen: convertClickhouseDateToJs(data.lastSeen),
+        firstSeen: convertClickhouseDateToJs(data.firstSeen),
+      };
+    });
 }
 
 export async function getProfileById(id: string, projectId: string) {
@@ -259,7 +273,7 @@ export function transformProfile({
     lastName: last_name,
     isExternal: profile.is_external,
     properties: toObject(profile.properties),
-    createdAt: new Date(created_at),
+    createdAt: convertClickhouseDateToJs(created_at),
     projectId: profile.project_id,
     id: profile.id,
     email: profile.email,

packages/db/test.ts

@@ -1,38 +0,0 @@
-import { conversionService } from './src/services/conversion.service';
-// 68/37
-async function main() {
-  const conversion = await conversionService.getConversion({
-    projectId: 'kiddokitchen-app',
-    startDate: '2025-02-01',
-    endDate: '2025-03-01',
-    funnelGroup: 'session_id',
-    breakdowns: [
-      {
-        name: 'os',
-      },
-    ],
-    interval: 'day',
-    events: [
-      {
-        segment: 'event',
-        name: 'screen_view',
-        filters: [
-          {
-            name: 'path',
-            operator: 'is',
-            value: ['Start'],
-          },
-        ],
-      },
-      {
-        segment: 'event',
-        name: 'sign_up',
-        filters: [],
-      },
-    ],
-  });
-
-  console.dir(conversion, { depth: null });
-}
-
-main();

packages/trpc/src/routers/chart.ts

@@ -322,9 +322,9 @@ export const chartRouter = createTRPCRouter({
     const previousPeriod = getChartPrevStartEndDate(currentPeriod);
 
     const [current, previous] = await Promise.all([
-      funnelService.getFunnel({ ...input, ...currentPeriod }),
+      funnelService.getFunnel({ ...input, ...currentPeriod, timezone }),
       input.previous
-        ? funnelService.getFunnel({ ...input, ...previousPeriod })
+        ? funnelService.getFunnel({ ...input, ...previousPeriod, timezone })
         : Promise.resolve(null),
     ]);
 
@@ -340,9 +340,13 @@ export const chartRouter = createTRPCRouter({
     const previousPeriod = getChartPrevStartEndDate(currentPeriod);
 
     const [current, previous] = await Promise.all([
-      conversionService.getConversion({ ...input, ...currentPeriod }),
+      conversionService.getConversion({ ...input, ...currentPeriod, timezone }),
       input.previous
-        ? conversionService.getConversion({ ...input, ...previousPeriod })
+        ? conversionService.getConversion({
+            ...input,
+            ...previousPeriod,
+            timezone,
+          })
         : Promise.resolve(null),
     ]);