fix: handle revenue better on overview (and remove it from top pages)

packages/db/src/services/overview.service.ts

@@ -7,6 +7,28 @@ import { TABLE_NAMES, ch } from '../clickhouse/client';
 import { clix } from '../clickhouse/query-builder';
 import { getEventFiltersWhereClause } from './chart.service';
 
+// Constants
+const ROLLUP_DATE_PREFIX = '1970-01-01';
+
+const COLUMN_PREFIX_MAP: Record<string, string> = {
+  region: 'country',
+  city: 'country',
+  browser_version: 'browser',
+  os_version: 'os',
+};
+
+// Types
+type MetricsRow = {
+  bounce_rate: number;
+  unique_visitors: number;
+  total_sessions: number;
+  avg_session_duration: number;
+  total_screen_views: number;
+  views_per_session: number;
+};
+
+type MetricsSeriesRow = MetricsRow & { date: string; total_revenue: number };
+
 export const zGetMetricsInput = z.object({
   projectId: z.string(),
   filters: z.array(z.any()),
@@ -85,11 +107,112 @@ export type IGetTopGenericInput = z.infer<typeof zGetTopGenericInput> & {
 export class OverviewService {
   constructor(private client: typeof ch) {}
 
+  // Helper methods
+  private isRollupRow(date: string): boolean {
+    return date.startsWith(ROLLUP_DATE_PREFIX);
+  }
+
+  private getFillConfig(interval: string, startDate: string, endDate: string) {
+    const useDateOnly = ['month', 'week'].includes(interval);
+    return {
+      from: clix.toStartOf(
+        clix.datetime(startDate, useDateOnly ? 'toDate' : 'toDateTime'),
+        interval as any,
+      ),
+      to: clix.datetime(endDate, useDateOnly ? 'toDate' : 'toDateTime'),
+      step: clix.toInterval('1', interval as any),
+    };
+  }
+
+  private createRevenueQuery({
+    projectId,
+    startDate,
+    endDate,
+    interval,
+    timezone,
+    filters,
+  }: {
+    projectId: string;
+    startDate: string;
+    endDate: string;
+    interval: string;
+    timezone: string;
+    filters: IChartEventFilter[];
+  }) {
+    return clix(this.client, timezone)
+      .select<{ date: string; total_revenue: number }>([
+        `${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
+        'sum(revenue) AS total_revenue',
+      ])
+      .from(TABLE_NAMES.events)
+      .where('project_id', '=', projectId)
+      .where('name', '=', 'revenue')
+      .where('revenue', '>', 0)
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ])
+      .rawWhere(this.getRawWhereClause('events', filters))
+      .groupBy(['date'])
+      .rollup()
+      .transform({
+        date: (item) => new Date(item.date).toISOString(),
+      });
+  }
+
+  private mergeRevenueIntoSeries<T extends { date: string }>(
+    series: T[],
+    revenueData: { date: string; total_revenue: number }[],
+  ): (T & { total_revenue: number })[] {
+    const revenueByDate = new Map(
+      revenueData
+        .filter((r) => !this.isRollupRow(r.date))
+        .map((r) => [r.date, r.total_revenue]),
+    );
+    return series.map((row) => ({
+      ...row,
+      total_revenue: revenueByDate.get(row.date) ?? 0,
+    }));
+  }
+
+  private getOverallRevenue(
+    revenueData: { date: string; total_revenue: number }[],
+  ): number {
+    return (
+      revenueData.find((r) => this.isRollupRow(r.date))?.total_revenue ?? 0
+    );
+  }
+
+  private withDistinctSessionsIfNeeded<T>(
+    query: ReturnType<typeof clix>,
+    params: {
+      filters: IChartEventFilter[];
+      projectId: string;
+      startDate: string;
+      endDate: string;
+      timezone: string;
+    },
+  ): ReturnType<typeof clix> {
+    if (!this.isPageFilter(params.filters)) {
+      query.rawWhere(this.getRawWhereClause('sessions', params.filters));
+      return query;
+    }
+
+    return clix(this.client, params.timezone)
+      .with('distinct_sessions', this.getDistinctSessions(params))
+      .merge(query)
+      .where(
+        'id',
+        'IN',
+        clix.exp('(SELECT session_id FROM distinct_sessions)'),
+      );
+  }
+
   isPageFilter(filters: IChartEventFilter[]) {
     return filters.some((filter) => filter.name === 'path' && filter.value);
   }
 
-  getMetrics({
+  async getMetrics({
     projectId,
     filters,
     startDate,
@@ -116,15 +239,128 @@ export class OverviewService {
       views_per_session: number;
       total_revenue: number;
     }[];
+  }> {
+    return this.isPageFilter(filters)
+      ? this.getMetricsWithPageFilter({
+          projectId,
+          filters,
+          startDate,
+          endDate,
+          interval,
+          timezone,
+        })
+      : this.getMetricsFromSessions({
+          projectId,
+          filters,
+          startDate,
+          endDate,
+          interval,
+          timezone,
+        });
+  }
+
+  private async getMetricsFromSessions({
+    projectId,
+    filters,
+    startDate,
+    endDate,
+    interval,
+    timezone,
+  }: IGetMetricsInput): Promise<{
+    metrics: MetricsRow & { total_revenue: number };
+    series: MetricsSeriesRow[];
   }> {
     const where = this.getRawWhereClause('sessions', filters);
-    if (this.isPageFilter(filters)) {
+    const fillConfig = this.getFillConfig(interval, startDate, endDate);
+
+    // Session metrics query
+    const sessionQuery = clix(this.client, timezone)
+      .select<{
+        date: string;
+        bounce_rate: number;
+        unique_visitors: number;
+        total_sessions: number;
+        avg_session_duration: number;
+        total_screen_views: number;
+        views_per_session: number;
+      }>([
+        `${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
+        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
+        'uniqIf(profile_id, sign > 0) AS unique_visitors',
+        'sum(sign) AS total_sessions',
+        'round(avgIf(duration, duration > 0 AND sign > 0), 2) / 1000 AS _avg_session_duration',
+        'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
+        'sum(sign * screen_view_count) AS total_screen_views',
+        'round(sum(sign * screen_view_count) * 1.0 / sum(sign), 2) AS views_per_session',
+      ])
+      .from('sessions')
+      .where('created_at', 'BETWEEN', [
+        clix.datetime(startDate, 'toDateTime'),
+        clix.datetime(endDate, 'toDateTime'),
+      ])
+      .where('project_id', '=', projectId)
+      .rawWhere(where)
+      .groupBy(['date'])
+      .having('sum(sign)', '>', 0)
+      .rollup()
+      .orderBy('date', 'ASC')
+      .fill(fillConfig.from, fillConfig.to, fillConfig.step)
+      .transform({
+        date: (item) => new Date(item.date).toISOString(),
+      });
+
+    // Revenue query
+    const revenueQuery = this.createRevenueQuery({
+      projectId,
+      startDate,
+      endDate,
+      interval,
+      timezone,
+      filters,
+    });
+
+    // Execute both queries in parallel and merge results
+    const [sessionRes, revenueRes] = await Promise.all([
+      sessionQuery.execute(),
+      revenueQuery.execute(),
+    ]);
+
+    const overallRevenue = this.getOverallRevenue(revenueRes);
+    const series = this.mergeRevenueIntoSeries(sessionRes.slice(1), revenueRes);
+
+    return {
+      metrics: {
+        bounce_rate: sessionRes[0]?.bounce_rate ?? 0,
+        unique_visitors: sessionRes[0]?.unique_visitors ?? 0,
+        total_sessions: sessionRes[0]?.total_sessions ?? 0,
+        avg_session_duration: sessionRes[0]?.avg_session_duration ?? 0,
+        total_screen_views: sessionRes[0]?.total_screen_views ?? 0,
+        views_per_session: sessionRes[0]?.views_per_session ?? 0,
+        total_revenue: overallRevenue,
+      },
+      series,
+    };
+  }
+
+  private async getMetricsWithPageFilter({
+    projectId,
+    filters,
+    startDate,
+    endDate,
+    interval,
+    timezone,
+  }: IGetMetricsInput): Promise<{
+    metrics: MetricsRow & { total_revenue: number };
+    series: MetricsSeriesRow[];
+  }> {
+    const where = this.getRawWhereClause('sessions', filters);
+    const fillConfig = this.getFillConfig(interval, startDate, endDate);
+
     // Session aggregation with bounce rates
     const sessionAggQuery = clix(this.client, timezone)
       .select([
-          `${clix.toStartOf('created_at', interval, timezone)} AS date`,
+        `${clix.toStartOf('created_at', interval as any, timezone)} AS date`,
         'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
-          'sum(revenue * sign) AS total_revenue',
       ])
       .from(TABLE_NAMES.sessions, true)
       .where('sign', '=', 1)
@@ -156,10 +392,11 @@ export class OverviewService {
     // Use toDate for month/week intervals, toDateTime for others
     const rollupDate =
       interval === 'month' || interval === 'week'
-          ? clix.date('1970-01-01')
+        ? clix.date(ROLLUP_DATE_PREFIX)
-          : clix.datetime('1970-01-01 00:00:00');
+        : clix.datetime(`${ROLLUP_DATE_PREFIX} 00:00:00`);
 
-      return clix(this.client, timezone)
+    // Main metrics query (without revenue)
+    const mainQuery = clix(this.client, timezone)
       .with('session_agg', sessionAggQuery)
       .with(
         'overall_bounce_rate',
@@ -169,16 +406,9 @@ export class OverviewService {
           .where('date', '=', rollupDate),
       )
       .with(
-          'overall_total_revenue',
+        'daily_session_stats',
         clix(this.client, timezone)
-            .select(['total_revenue'])
+          .select(['date', 'bounce_rate'])
-            .from('session_agg')
-            .where('date', '=', rollupDate),
-        )
-        .with(
-          'daily_stats',
-          clix(this.client, timezone)
-            .select(['date', 'bounce_rate', 'total_revenue'])
           .from('session_agg')
           .where('date', '!=', rollupDate),
       )
@@ -191,30 +421,26 @@ export class OverviewService {
         avg_session_duration: number;
         total_screen_views: number;
         views_per_session: number;
-          total_revenue: number;
         overall_unique_visitors: number;
         overall_total_sessions: number;
         overall_bounce_rate: number;
-          overall_total_revenue: number;
       }>([
-          `${clix.toStartOf('e.created_at', interval)} AS date`,
+        `${clix.toStartOf('e.created_at', interval as any)} AS date`,
-          'ds.bounce_rate as bounce_rate',
+        'dss.bounce_rate as bounce_rate',
         'uniq(e.profile_id) AS unique_visitors',
         'uniq(e.session_id) AS total_sessions',
         'round(avgIf(duration, duration > 0), 2) / 1000 AS _avg_session_duration',
         'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
         'count(*) AS total_screen_views',
         'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
-          'ds.total_revenue AS total_revenue',
         '(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
         '(SELECT total_sessions FROM overall_unique_visitors) AS overall_total_sessions',
         '(SELECT bounce_rate FROM overall_bounce_rate) AS overall_bounce_rate',
-          '(SELECT total_revenue FROM overall_total_revenue) AS overall_total_revenue',
       ])
       .from(`${TABLE_NAMES.events} AS e`)
       .leftJoin(
-          'daily_stats AS ds',
+        'daily_session_stats AS dss',
-          `${clix.toStartOf('e.created_at', interval)} = ds.date`,
+        `${clix.toStartOf('e.created_at', interval as any)} = dss.date`,
       )
       .where('e.project_id', '=', projectId)
       .where('e.name', '=', 'screen_view')
@@ -223,129 +449,55 @@ export class OverviewService {
         clix.datetime(endDate, 'toDateTime'),
       ])
       .rawWhere(this.getRawWhereClause('events', filters))
-        .groupBy(['date', 'ds.bounce_rate', 'ds.total_revenue'])
+      .groupBy(['date', 'dss.bounce_rate'])
       .orderBy('date', 'ASC')
-        .fill(
+      .fill(fillConfig.from, fillConfig.to, fillConfig.step)
-          clix.toStartOf(
-            clix.datetime(
-              startDate,
-              ['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
-            ),
-            interval,
-          ),
-          clix.datetime(
-            endDate,
-            ['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
-          ),
-          clix.toInterval('1', interval),
-        )
       .transform({
         date: (item) => new Date(item.date).toISOString(),
-        })
+      });
-        .execute()
+
-        .then((res) => {
+    // Revenue query
-          const anyRowWithData = res.find(
+    const revenueQuery = this.createRevenueQuery({
+      projectId,
+      startDate,
+      endDate,
+      interval,
+      timezone,
+      filters,
+    });
+
+    // Execute both queries in parallel and merge results
+    const [mainRes, revenueRes] = await Promise.all([
+      mainQuery.execute(),
+      revenueQuery.execute(),
+    ]);
+
+    const overallRevenue = this.getOverallRevenue(revenueRes);
+    const series = this.mergeRevenueIntoSeries(mainRes, revenueRes);
+
+    const anyRowWithData = mainRes.find(
       (item) =>
         item.overall_bounce_rate !== null ||
         item.overall_total_sessions !== null ||
-              item.overall_unique_visitors !== null ||
+        item.overall_unique_visitors !== null,
-              item.overall_total_revenue !== null,
     );
+
     return {
       metrics: {
         bounce_rate: anyRowWithData?.overall_bounce_rate ?? 0,
         unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0,
         total_sessions: anyRowWithData?.overall_total_sessions ?? 0,
         avg_session_duration: average(
-                res.map((item) => item.avg_session_duration),
+          mainRes.map((item) => item.avg_session_duration),
-              ),
-              total_screen_views: sum(
-                res.map((item) => item.total_screen_views),
         ),
+        total_screen_views: sum(mainRes.map((item) => item.total_screen_views)),
         views_per_session: average(
-                res.map((item) => item.views_per_session),
+          mainRes.map((item) => item.views_per_session),
         ),
-              total_revenue: anyRowWithData?.overall_total_revenue ?? 0,
+        total_revenue: overallRevenue,
       },
-            series: res.map(
+      series,
-              omit([
-                'overall_bounce_rate',
-                'overall_unique_visitors',
-                'overall_total_sessions',
-                'overall_total_revenue',
-              ]),
-            ),
     };
-        });
-    }
-
-    const query = clix(this.client, timezone)
-      .select<{
-        date: string;
-        bounce_rate: number;
-        unique_visitors: number;
-        total_sessions: number;
-        avg_session_duration: number;
-        total_screen_views: number;
-        views_per_session: number;
-        total_revenue: number;
-      }>([
-        `${clix.toStartOf('created_at', interval, timezone)} AS date`,
-        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
-        'uniqIf(profile_id, sign > 0) AS unique_visitors',
-        'sum(sign) AS total_sessions',
-        'round(avgIf(duration, duration > 0 AND sign > 0), 2) / 1000 AS _avg_session_duration',
-        'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
-        'sum(sign * screen_view_count) AS total_screen_views',
-        'round(sum(sign * screen_view_count) * 1.0 / sum(sign), 2) AS views_per_session',
-        'sum(revenue * sign) AS total_revenue',
-      ])
-      .from('sessions')
-      .where('created_at', 'BETWEEN', [
-        clix.datetime(startDate, 'toDateTime'),
-        clix.datetime(endDate, 'toDateTime'),
-      ])
-      .where('project_id', '=', projectId)
-      .rawWhere(where)
-      .groupBy(['date'])
-      .having('sum(sign)', '>', 0)
-      .rollup()
-      .orderBy('date', 'ASC')
-      .fill(
-        clix.toStartOf(
-          clix.datetime(
-            startDate,
-            ['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
-          ),
-          interval,
-        ),
-        clix.datetime(
-          endDate,
-          ['month', 'week'].includes(interval) ? 'toDate' : 'toDateTime',
-        ),
-        clix.toInterval('1', interval),
-      )
-      .transform({
-        date: (item) => new Date(item.date).toISOString(),
-      });
-
-    return query.execute().then((res) => {
-      // First row is the rollup row containing the total values
-      return {
-        metrics: {
-          bounce_rate: res[0]?.bounce_rate ?? 0,
-          unique_visitors: res[0]?.unique_visitors ?? 0,
-          total_sessions: res[0]?.total_sessions ?? 0,
-          avg_session_duration: res[0]?.avg_session_duration ?? 0,
-          total_screen_views: res[0]?.total_screen_views ?? 0,
-          views_per_session: res[0]?.views_per_session ?? 0,
-          total_revenue: res[0]?.total_revenue ?? 0,
-        },
-        series: res
-          .slice(1)
-          .map(omit(['overall_bounce_rate', 'overall_unique_visitors'])),
-      };
-    });
   }
 
   getRawWhereClause(type: 'events' | 'sessions', filters: IChartEventFilter[]) {
@@ -368,12 +520,6 @@ export class OverviewService {
         }
         return item;
       }),
-      // .filter((item) => {
-      //   if (this.isPageFilter(filters) && type === 'sessions') {
-      //     return item.name !== 'entry_path' && item.name !== 'entry_origin';
-      //   }
-      //   return true;
-      // }),
     );
 
     return Object.values(where).join(' AND ');
@@ -414,7 +560,6 @@ export class OverviewService {
         'entry_path',
         'entry_origin',
         'coalesce(round(countIf(is_bounce = 1 AND sign = 1) * 100.0 / countIf(sign = 1), 2), 0) as bounce_rate',
-        'sum(revenue * sign) as revenue',
       ])
       .from(TABLE_NAMES.sessions, true)
       .where('sign', '=', 1)
@@ -446,7 +591,6 @@ export class OverviewService {
         'p.avg_duration',
         'p.count as sessions',
         'b.bounce_rate',
-        'coalesce(b.revenue, 0) as revenue',
       ])
       .from('page_stats p', false)
       .leftJoin(
@@ -469,16 +613,6 @@ export class OverviewService {
     limit = 10,
     timezone,
   }: IGetTopEntryExitInput) {
-    const where = this.getRawWhereClause('sessions', filters);
-
-    const distinctSessionQuery = this.getDistinctSessions({
-      projectId,
-      filters,
-      startDate,
-      endDate,
-      timezone,
-    });
-
     const offset = (cursor - 1) * limit;
 
     const query = clix(this.client, timezone)
@@ -503,25 +637,19 @@ export class OverviewService {
         clix.datetime(startDate, 'toDateTime'),
         clix.datetime(endDate, 'toDateTime'),
       ])
-      .rawWhere(where)
       .groupBy([`${mode}_origin`, `${mode}_path`])
       .having('sum(sign)', '>', 0)
       .orderBy('sessions', 'DESC')
       .limit(limit)
       .offset(offset);
 
-    let mainQuery = query;
+    const mainQuery = this.withDistinctSessionsIfNeeded(query, {
-
+      projectId,
-    if (this.isPageFilter(filters)) {
+      filters,
-      mainQuery = clix(this.client, timezone)
+      startDate,
-        .with('distinct_sessions', distinctSessionQuery)
+      endDate,
-        .merge(query)
+      timezone,
-        .where(
+    });
-          'id',
-          'IN',
-          clix.exp('(SELECT session_id FROM distinct_sessions)'),
-        );
-    }
 
     return mainQuery.execute();
   }
@@ -560,28 +688,7 @@ export class OverviewService {
     limit = 10,
     timezone,
   }: IGetTopGenericInput) {
-    const distinctSessionQuery = this.getDistinctSessions({
+    const prefixColumn = COLUMN_PREFIX_MAP[column] ?? null;
-      projectId,
-      filters,
-      startDate,
-      endDate,
-      timezone,
-    });
-
-    const prefixColumn = (() => {
-      switch (column) {
-        case 'region':
-          return 'country';
-        case 'city':
-          return 'country';
-        case 'browser_version':
-          return 'browser';
-        case 'os_version':
-          return 'os';
-      }
-      return null;
-    })();
-
     const offset = (cursor - 1) * limit;
 
     const query = clix(this.client, timezone)
@@ -612,24 +719,15 @@ export class OverviewService {
       .limit(limit)
       .offset(offset);
 
-    let mainQuery = query;
+    const mainQuery = this.withDistinctSessionsIfNeeded(query, {
+      projectId,
+      filters,
+      startDate,
+      endDate,
+      timezone,
+    });
 
-    if (this.isPageFilter(filters)) {
+    return mainQuery.execute();
-      mainQuery = clix(this.client, timezone)
-        .with('distinct_sessions', distinctSessionQuery)
-        .merge(query)
-        .where(
-          'id',
-          'IN',
-          clix.exp('(SELECT session_id FROM distinct_sessions)'),
-        );
-    } else {
-      mainQuery.rawWhere(this.getRawWhereClause('sessions', filters));
-    }
-
-    const res = await mainQuery.execute();
-
-    return res;
   }
 }