feat: revenue tracking

* wip * wip * wip * wip * show revenue better on overview * align realtime and overview counters * update revenue docs * always return device id * add project settings, improve projects charts, * fix: comments * fixes * fix migration * ignore sql files * fix comments
2025-11-19 14:27:34 +01:00
parent d61cbf6f2c
commit 790801b728
58 changed files with 2191 additions and 23691 deletions
--- a/packages/db/src/services/chart.service.ts
+++ b/packages/db/src/services/chart.service.ts
@@ -174,23 +174,43 @@ export function getChartSql({
  }

  if (event.segment === 'property_sum' && event.property) {
-    sb.select.count = `sum(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
-    sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    if (event.property === 'revenue') {
+      sb.select.count = `sum(revenue) as count`;
+      sb.where.property = `revenue > 0`;
+    } else {
+      sb.select.count = `sum(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
+      sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    }
  }

  if (event.segment === 'property_average' && event.property) {
-    sb.select.count = `avg(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
-    sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    if (event.property === 'revenue') {
+      sb.select.count = `avg(revenue) as count`;
+      sb.where.property = `revenue > 0`;
+    } else {
+      sb.select.count = `avg(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
+      sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    }
  }

  if (event.segment === 'property_max' && event.property) {
-    sb.select.count = `max(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
-    sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    if (event.property === 'revenue') {
+      sb.select.count = `max(revenue) as count`;
+      sb.where.property = `revenue > 0`;
+    } else {
+      sb.select.count = `max(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
+      sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    }
  }

  if (event.segment === 'property_min' && event.property) {
-    sb.select.count = `min(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
-    sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    if (event.property === 'revenue') {
+      sb.select.count = `min(revenue) as count`;
+      sb.where.property = `revenue > 0`;
+    } else {
+      sb.select.count = `min(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
+      sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
+    }
  }

  if (event.segment === 'one_event_per_user') {
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -25,6 +25,7 @@ import {
  getProfilesCached,
  upsertProfile,
 } from './profile.service';
+import type { IClickhouseSession } from './session.service';

 export type IImportedEvent = Omit<
  IClickhouseEvent,
@@ -92,12 +93,62 @@ export interface IClickhouseEvent {
  imported_at: string | null;
  sdk_name: string;
  sdk_version: string;
+  revenue?: number;

  // They do not exist here. Just make ts happy for now
  profile?: IServiceProfile;
  meta?: EventMeta;
 }

+export function transformSessionToEvent(
+  session: IClickhouseSession,
+): IServiceEvent {
+  return {
+    id: '', // Not used
+    name: 'screen_view',
+    sessionId: session.id,
+    profileId: session.profile_id,
+    path: session.exit_path,
+    origin: session.exit_origin,
+    createdAt: convertClickhouseDateToJs(session.ended_at),
+    referrer: session.referrer,
+    referrerName: session.referrer_name,
+    referrerType: session.referrer_type,
+    os: session.os,
+    osVersion: session.os_version,
+    browser: session.browser,
+    browserVersion: session.browser_version,
+    device: session.device,
+    brand: session.brand,
+    model: session.model,
+    country: session.country,
+    region: session.region,
+    city: session.city,
+    longitude: session.longitude,
+    latitude: session.latitude,
+    projectId: session.project_id,
+    deviceId: session.device_id,
+    duration: 0,
+    revenue: session.revenue,
+    properties: {
+      ...session.properties,
+      is_bounce: session.is_bounce,
+      __query: {
+        utm_medium: session.utm_medium,
+        utm_source: session.utm_source,
+        utm_campaign: session.utm_campaign,
+        utm_content: session.utm_content,
+        utm_term: session.utm_term,
+      },
+    },
+    profile: undefined,
+    meta: undefined,
+    importedAt: undefined,
+    sdkName: undefined,
+    sdkVersion: undefined,
+  };
+}
+
 export function transformEvent(event: IClickhouseEvent): IServiceEvent {
  return {
    id: event.id,
@@ -131,6 +182,7 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
    sdkName: event.sdk_name,
    sdkVersion: event.sdk_version,
    profile: event.profile,
+    revenue: event.revenue,
  };
 }

@@ -178,6 +230,7 @@ export interface IServiceEvent {
  meta: EventMeta | undefined;
  sdkName: string | undefined;
  sdkVersion: string | undefined;
+  revenue?: number;
 }

 type SelectHelper<T> = {
@@ -336,6 +389,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
    imported_at: null,
    sdk_name: payload.sdkName ?? '',
    sdk_version: payload.sdkVersion ?? '',
+    revenue: payload.revenue,
  };

  const promises = [sessionBuffer.add(event), eventBuffer.add(event)];
--- a/packages/db/src/services/overview.service.ts
+++ b/packages/db/src/services/overview.service.ts
@@ -104,6 +104,7 @@ export class OverviewService {
      avg_session_duration: number;
      total_screen_views: number;
      views_per_session: number;
+      total_revenue: number;
    };
    series: {
      date: string;
@@ -113,6 +114,7 @@ export class OverviewService {
      avg_session_duration: number;
      total_screen_views: number;
      views_per_session: number;
+      total_revenue: number;
    }[];
  }> {
    const where = this.getRawWhereClause('sessions', filters);
@@ -122,6 +124,7 @@ export class OverviewService {
        .select([
          `${clix.toStartOf('created_at', interval, timezone)} AS date`,
          'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
+          'sum(revenue * sign) AS total_revenue',
        ])
        .from(TABLE_NAMES.sessions, true)
        .where('sign', '=', 1)
@@ -165,10 +168,17 @@ export class OverviewService {
            .from('session_agg')
            .where('date', '=', rollupDate),
        )
+        .with(
+          'overall_total_revenue',
+          clix(this.client, timezone)
+            .select(['total_revenue'])
+            .from('session_agg')
+            .where('date', '=', rollupDate),
+        )
        .with(
          'daily_stats',
          clix(this.client, timezone)
-            .select(['date', 'bounce_rate'])
+            .select(['date', 'bounce_rate', 'total_revenue'])
            .from('session_agg')
            .where('date', '!=', rollupDate),
        )
@@ -181,9 +191,11 @@ export class OverviewService {
          avg_session_duration: number;
          total_screen_views: number;
          views_per_session: number;
+          total_revenue: number;
          overall_unique_visitors: number;
          overall_total_sessions: number;
          overall_bounce_rate: number;
+          overall_total_revenue: number;
        }>([
          `${clix.toStartOf('e.created_at', interval)} AS date`,
          'ds.bounce_rate as bounce_rate',
@@ -193,9 +205,11 @@ export class OverviewService {
          'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
          'count(*) AS total_screen_views',
          'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
+          'ds.total_revenue AS total_revenue',
          '(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
          '(SELECT total_sessions FROM overall_unique_visitors) AS overall_total_sessions',
          '(SELECT bounce_rate FROM overall_bounce_rate) AS overall_bounce_rate',
+          '(SELECT total_revenue FROM overall_total_revenue) AS overall_total_revenue',
        ])
        .from(`${TABLE_NAMES.events} AS e`)
        .leftJoin(
@@ -209,7 +223,7 @@ export class OverviewService {
          clix.datetime(endDate, 'toDateTime'),
        ])
        .rawWhere(this.getRawWhereClause('events', filters))
-        .groupBy(['date', 'ds.bounce_rate'])
+        .groupBy(['date', 'ds.bounce_rate', 'ds.total_revenue'])
        .orderBy('date', 'ASC')
        .fill(
          clix.toStartOf(
@@ -234,7 +248,8 @@ export class OverviewService {
            (item) =>
              item.overall_bounce_rate !== null ||
              item.overall_total_sessions !== null ||
-              item.overall_unique_visitors !== null,
+              item.overall_unique_visitors !== null ||
+              item.overall_total_revenue !== null,
          );
          return {
            metrics: {
@@ -250,12 +265,14 @@ export class OverviewService {
              views_per_session: average(
                res.map((item) => item.views_per_session),
              ),
+              total_revenue: anyRowWithData?.overall_total_revenue ?? 0,
            },
            series: res.map(
              omit([
                'overall_bounce_rate',
                'overall_unique_visitors',
                'overall_total_sessions',
+                'overall_total_revenue',
              ]),
            ),
          };
@@ -271,6 +288,7 @@ export class OverviewService {
        avg_session_duration: number;
        total_screen_views: number;
        views_per_session: number;
+        total_revenue: number;
      }>([
        `${clix.toStartOf('created_at', interval, timezone)} AS date`,
        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
@@ -280,6 +298,7 @@ export class OverviewService {
        'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
        'sum(sign * screen_view_count) AS total_screen_views',
        'round(sum(sign * screen_view_count) * 1.0 / sum(sign), 2) AS views_per_session',
+        'sum(revenue * sign) AS total_revenue',
      ])
      .from('sessions')
      .where('created_at', 'BETWEEN', [
@@ -320,6 +339,7 @@ export class OverviewService {
          avg_session_duration: res[0]?.avg_session_duration ?? 0,
          total_screen_views: res[0]?.total_screen_views ?? 0,
          views_per_session: res[0]?.views_per_session ?? 0,
+          total_revenue: res[0]?.total_revenue ?? 0,
        },
        series: res
          .slice(1)
@@ -394,6 +414,7 @@ export class OverviewService {
        'entry_path',
        'entry_origin',
        'coalesce(round(countIf(is_bounce = 1 AND sign = 1) * 100.0 / countIf(sign = 1), 2), 0) as bounce_rate',
+        'sum(revenue * sign) as revenue',
      ])
      .from(TABLE_NAMES.sessions, true)
      .where('sign', '=', 1)
@@ -417,6 +438,7 @@ export class OverviewService {
        avg_duration: number;
        bounce_rate: number;
        sessions: number;
+        revenue: number;
      }>([
        'p.title',
        'p.origin',
@@ -424,6 +446,7 @@ export class OverviewService {
        'p.avg_duration',
        'p.count as sessions',
        'b.bounce_rate',
+        'coalesce(b.revenue, 0) as revenue',
      ])
      .from('page_stats p', false)
      .leftJoin(
@@ -465,12 +488,14 @@ export class OverviewService {
        avg_duration: number;
        bounce_rate: number;
        sessions: number;
+        revenue: number;
      }>([
        `${mode}_origin AS origin`,
        `${mode}_path AS path`,
        'round(avg(duration * sign)/1000, 2) as avg_duration',
        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
        'sum(sign) as sessions',
+        'sum(revenue * sign) as revenue',
      ])
      .from(TABLE_NAMES.sessions, true)
      .where('project_id', '=', projectId)
@@ -566,12 +591,14 @@ export class OverviewService {
        sessions: number;
        bounce_rate: number;
        avg_session_duration: number;
+        revenue: number;
      }>([
        prefixColumn && `${prefixColumn} as prefix`,
        `nullIf(${column}, '') as name`,
        'sum(sign) as sessions',
        'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) AS bounce_rate',
        'round(avgIf(duration, duration > 0 AND sign > 0), 2)/1000 AS avg_session_duration',
+        'sum(revenue * sign) as revenue',
      ])
      .from(TABLE_NAMES.sessions, true)
      .where('project_id', '=', projectId)
--- a/packages/db/src/services/profile.service.ts
+++ b/packages/db/src/services/profile.service.ts
@@ -28,6 +28,7 @@ export type IProfileMetrics = {
  avgEventsPerSession: number;
  conversionEvents: number;
  avgTimeBetweenSessions: number;
+  revenue: number;
 };
 export function getProfileMetrics(profileId: string, projectId: string) {
  return chQuery<
@@ -76,6 +77,9 @@ export function getProfileMetrics(profileId: string, projectId: string) {
          WHEN (SELECT sessions FROM sessions) <= 1 THEN 0
          ELSE round(dateDiff('second', (SELECT firstSeen FROM firstSeen), (SELECT lastSeen FROM lastSeen)) / nullIf((SELECT sessions FROM sessions) - 1, 0), 1)
        END as avgTimeBetweenSessions
+    ),
+    revenue AS (
+      SELECT sum(revenue) as revenue FROM ${TABLE_NAMES.events} WHERE name = 'revenue' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
    )
    SELECT 
      (SELECT lastSeen FROM lastSeen) as lastSeen, 
@@ -89,7 +93,8 @@ export function getProfileMetrics(profileId: string, projectId: string) {
      (SELECT bounceRate FROM bounceRate) as bounceRate,
      (SELECT avgEventsPerSession FROM avgEventsPerSession) as avgEventsPerSession,
      (SELECT conversionEvents FROM conversionEvents) as conversionEvents,
-      (SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions
+      (SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions,
+      (SELECT revenue FROM revenue) as revenue
  `)
    .then((data) => data[0]!)
    .then((data) => {