feat: revenue tracking

* wip * wip * wip * wip * show revenue better on overview * align realtime and overview counters * update revenue docs * always return device id * add project settings, improve projects charts, * fix: comments * fixes * fix migration * ignore sql files * fix comments
2025-11-19 14:27:34 +01:00
parent d61cbf6f2c
commit 790801b728
58 changed files with 2191 additions and 23691 deletions
--- a/apps/worker/src/jobs/events.create-session-end.ts
+++ b/apps/worker/src/jobs/events.create-session-end.ts
@@ -15,6 +15,7 @@ import {
  getHasFunnelRules,
  getNotificationRulesByProjectId,
  sessionBuffer,
+  transformSessionToEvent,
 } from '@openpanel/db';
 import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';

@@ -31,7 +32,7 @@ async function getSessionEvents({
  sessionId: string;
  startAt: Date;
  endAt: Date;
-}): Promise<ReturnType<typeof getEvents>> {
+}): Promise<IServiceEvent[]> {
  const sql = `
    SELECT * FROM ${TABLE_NAMES.events} 
    WHERE 
@@ -42,16 +43,18 @@ async function getSessionEvents({
  `;

  const [lastScreenView, eventsInDb] = await Promise.all([
-    eventBuffer.getLastScreenView({
-      projectId,
+    sessionBuffer.getExistingSession({
      sessionId,
    }),
    getEvents(sql),
  ]);

  // sort last inserted first
-  return [lastScreenView, ...eventsInDb]
-    .filter((event): event is IServiceEvent => !!event)
+  return [
+    lastScreenView ? transformSessionToEvent(lastScreenView) : null,
+    ...eventsInDb,
+  ]
+    .flatMap((event) => (event ? [event] : []))
    .sort(
      (a, b) =>
        new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
@@ -69,7 +72,9 @@ export async function createSessionEnd(

  logger.debug('Processing session end job');

-  const session = await sessionBuffer.getExistingSession(payload.sessionId);
+  const session = await sessionBuffer.getExistingSession({
+    sessionId: payload.sessionId,
+  });

  if (!session) {
    throw new Error('Session not found');
@@ -86,26 +91,21 @@ export async function createSessionEnd(
    });
  }

-  const lastScreenView = await eventBuffer.getLastScreenView({
-    projectId: payload.projectId,
-    sessionId: payload.sessionId,
-  });
-
  // Create session end event
  return createEvent({
    ...payload,
    properties: {
      ...payload.properties,
-      ...(lastScreenView?.properties ?? {}),
+      ...(session?.properties ?? {}),
      __bounce: session.is_bounce,
    },
    name: 'session_end',
    duration: session.duration ?? 0,
-    path: lastScreenView?.path ?? '',
+    path: session.exit_path ?? '',
    createdAt: new Date(
-      convertClickhouseDateToJs(session.ended_at).getTime() + 100,
+      convertClickhouseDateToJs(session.ended_at).getTime() + 1000,
    ),
-    profileId: lastScreenView?.profileId || payload.profileId,
+    profileId: session.profile_id || payload.profileId,
  });
 }

--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -14,14 +14,14 @@ import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
 import {
  checkNotificationRulesForEvent,
  createEvent,
-  eventBuffer,
+  sessionBuffer,
 } from '@openpanel/db';
 import type { ILogger } from '@openpanel/logger';
 import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
 import * as R from 'ramda';
 import { v4 as uuid } from 'uuid';

-const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp'];
+const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];

 // This function will merge two objects.
 // First it will strip '' and undefined/null from B
@@ -41,6 +41,17 @@ async function createEventAndNotify(
  return event;
 }

+const parseRevenue = (revenue: unknown): number | undefined => {
+  if (!revenue) return undefined;
+  if (typeof revenue === 'number') return revenue;
+  if (typeof revenue === 'string') {
+    const parsed = Number.parseFloat(revenue);
+    if (Number.isNaN(parsed)) return undefined;
+    return parsed;
+  }
+  return undefined;
+};
+
 export async function incomingEvent(
  jobPayload: EventsQueuePayloadIncomingEvent['payload'],
 ) {
@@ -115,12 +126,16 @@ export async function incomingEvent(
    device: uaInfo.device,
    brand: uaInfo.brand,
    model: uaInfo.model,
+    revenue:
+      body.name === 'revenue' && '__revenue' in properties
+        ? parseRevenue(properties.__revenue)
+        : undefined,
  } as const;

  // if timestamp is from the past we dont want to create a new session
  if (uaInfo.isServer || isTimestampFromThePast) {
-    const screenView = profileId
-      ? await eventBuffer.getLastScreenView({
+    const session = profileId
+      ? await sessionBuffer.getExistingSession({
          profileId,
          projectId,
        })
@@ -128,25 +143,25 @@ export async function incomingEvent(

    const payload = {
      ...baseEvent,
-      deviceId: screenView?.deviceId ?? '',
-      sessionId: screenView?.sessionId ?? '',
-      referrer: screenView?.referrer ?? undefined,
-      referrerName: screenView?.referrerName ?? undefined,
-      referrerType: screenView?.referrerType ?? undefined,
-      path: screenView?.path ?? baseEvent.path,
-      os: screenView?.os ?? baseEvent.os,
-      osVersion: screenView?.osVersion ?? baseEvent.osVersion,
-      browserVersion: screenView?.browserVersion ?? baseEvent.browserVersion,
-      browser: screenView?.browser ?? baseEvent.browser,
-      device: screenView?.device ?? baseEvent.device,
-      brand: screenView?.brand ?? baseEvent.brand,
-      model: screenView?.model ?? baseEvent.model,
-      city: screenView?.city ?? baseEvent.city,
-      country: screenView?.country ?? baseEvent.country,
-      region: screenView?.region ?? baseEvent.region,
-      longitude: screenView?.longitude ?? baseEvent.longitude,
-      latitude: screenView?.latitude ?? baseEvent.latitude,
-      origin: screenView?.origin ?? baseEvent.origin,
+      deviceId: session?.device_id ?? '',
+      sessionId: session?.id ?? '',
+      referrer: session?.referrer ?? undefined,
+      referrerName: session?.referrer_name ?? undefined,
+      referrerType: session?.referrer_type ?? undefined,
+      path: session?.exit_path ?? baseEvent.path,
+      origin: session?.exit_origin ?? baseEvent.origin,
+      os: session?.os ?? baseEvent.os,
+      osVersion: session?.os_version ?? baseEvent.osVersion,
+      browserVersion: session?.browser_version ?? baseEvent.browserVersion,
+      browser: session?.browser ?? baseEvent.browser,
+      device: session?.device ?? baseEvent.device,
+      brand: session?.brand ?? baseEvent.brand,
+      model: session?.model ?? baseEvent.model,
+      city: session?.city ?? baseEvent.city,
+      country: session?.country ?? baseEvent.country,
+      region: session?.region ?? baseEvent.region,
+      longitude: session?.longitude ?? baseEvent.longitude,
+      latitude: session?.latitude ?? baseEvent.latitude,
    };

    return createEventAndNotify(payload as IServiceEvent, logger);
@@ -160,8 +175,7 @@ export async function incomingEvent(
  });

  const lastScreenView = sessionEnd
-    ? await eventBuffer.getLastScreenView({
-        projectId,
+    ? await sessionBuffer.getExistingSession({
        sessionId: sessionEnd.sessionId,
      })
    : null;
@@ -173,8 +187,8 @@ export async function incomingEvent(
    referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,
    referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,
    // if the path is not set, use the last screen view path
-    path: baseEvent.path || lastScreenView?.path || '',
-    origin: baseEvent.origin || lastScreenView?.origin || '',
+    path: baseEvent.path || lastScreenView?.exit_path || '',
+    origin: baseEvent.origin || lastScreenView?.exit_origin || '',
  } as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;

  if (!sessionEnd) {
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -1,4 +1,11 @@
-import { type IServiceEvent, createEvent } from '@openpanel/db';
+import {
+  type IClickhouseSession,
+  type IServiceEvent,
+  type IServiceSession,
+  createEvent,
+  formatClickhouseDate,
+  sessionBuffer,
+} from '@openpanel/db';
 import { eventBuffer } from '@openpanel/db';
 import {
  type EventsQueuePayloadIncomingEvent,
@@ -14,10 +21,9 @@ vi.mock('@openpanel/db', async () => {
  return {
    ...actual,
    createEvent: vi.fn(),
-    getLastScreenView: vi.fn(),
    checkNotificationRulesForEvent: vi.fn().mockResolvedValue(true),
-    eventBuffer: {
-      getLastScreenView: vi.fn(),
+    sessionBuffer: {
+      getExistingSession: vi.fn(),
    },
  };
 });
@@ -106,6 +112,7 @@ describe('incomingEvent', () => {
      country: 'US',
      city: 'New York',
      region: 'NY',
+      revenue: undefined,
      longitude: 0,
      latitude: 0,
      os: 'Windows',
@@ -210,6 +217,7 @@ describe('incomingEvent', () => {
      country: 'US',
      city: 'New York',
      region: 'NY',
+      revenue: undefined,
      longitude: 0,
      latitude: 0,
      os: 'Windows',
@@ -278,10 +286,47 @@ describe('incomingEvent', () => {
      referrerType: 'search',
    };

-    // Mock the eventBuffer.getLastScreenView method
-    vi.mocked(eventBuffer.getLastScreenView).mockResolvedValueOnce(
-      mockLastScreenView as IServiceEvent,
-    );
+    vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce({
+      id: 'last-session-456',
+      event_count: 0,
+      screen_view_count: 0,
+      entry_path: '/last-path',
+      entry_origin: 'https://example.com',
+      exit_path: '/last-path',
+      exit_origin: 'https://example.com',
+      created_at: formatClickhouseDate(timestamp),
+      ended_at: formatClickhouseDate(timestamp),
+      os: 'iOS',
+      os_version: '15.0',
+      browser: 'Safari',
+      browser_version: '15.0',
+      device: 'mobile',
+      brand: 'Apple',
+      model: 'iPhone',
+      country: 'CA',
+      region: 'ON',
+      city: 'Toronto',
+      longitude: 0,
+      latitude: 0,
+      duration: 0,
+      referrer: 'https://google.com',
+      referrer_name: 'Google',
+      referrer_type: 'search',
+      is_bounce: false,
+      utm_term: '',
+      utm_source: '',
+      utm_campaign: '',
+      utm_content: '',
+      utm_medium: '',
+      revenue: 0,
+      properties: {},
+      project_id: projectId,
+      device_id: 'last-device-123',
+      profile_id: 'profile-123',
+      screen_views: [],
+      sign: 1,
+      version: 1,
+    } satisfies IClickhouseSession);

    await incomingEvent(jobData);

@@ -317,6 +362,7 @@ describe('incomingEvent', () => {
      referrerType: 'search',
      sdkName: 'server',
      sdkVersion: '1.0.0',
+      revenue: undefined,
    });

    expect(sessionsQueue.add).not.toHaveBeenCalled();
@@ -345,9 +391,6 @@ describe('incomingEvent', () => {
      uaInfo: uaInfoServer,
    };

-    // Mock getLastScreenView to return null
-    vi.mocked(eventBuffer.getLastScreenView).mockResolvedValueOnce(null);
-
    await incomingEvent(jobData);

    expect((createEvent as Mock).mock.calls[0]![0]).toStrictEqual({
@@ -365,6 +408,7 @@ describe('incomingEvent', () => {
      country: 'US',
      city: 'New York',
      region: 'NY',
+      revenue: undefined,
      longitude: 0,
      latitude: 0,
      os: '',