zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: revenue tracking

Browse Source 
* wip

* wip

* wip

* wip

* show revenue better on overview

* align realtime and overview counters

* update revenue docs

* always return device id

* add project settings, improve projects charts,

* fix: comments

* fixes

* fix migration

* ignore sql files

* fix comments
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-19 14:27:34 +01:00
committed by GitHub
parent d61cbf6f2c
commit 790801b728
58 changed files with 2191 additions and 23691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
packages/db/code-migrations/3-init-ch.sql
View File

@@ -1,167 +0,0 @@
CREATE DATABASE IF NOT EXISTS openpanel;
---
CREATE TABLE IF NOT EXISTS self_hosting (
`created_at` Date,
`domain` String,
`count` UInt64
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (domain, created_at);
---
CREATE TABLE IF NOT EXISTS events (
`id` UUID DEFAULT generateUUIDv4(),
`name` LowCardinality(String),
`sdk_name` LowCardinality(String),
`sdk_version` LowCardinality(String),
`device_id` String CODEC(ZSTD(3)),
`profile_id` String CODEC(ZSTD(3)),
`project_id` String CODEC(ZSTD(3)),
`session_id` String CODEC(LZ4),
`path` String CODEC(ZSTD(3)),
`origin` String CODEC(ZSTD(3)),
`referrer` String CODEC(ZSTD(3)),
`referrer_name` String CODEC(ZSTD(3)),
`referrer_type` LowCardinality(String),
`duration` UInt64 CODEC(Delta(4), LZ4),
`properties` Map(String, String) CODEC(ZSTD(3)),
`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),
`country` LowCardinality(FixedString(2)),
`city` String,
`region` LowCardinality(String),
`longitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`latitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`os` LowCardinality(String),
`os_version` LowCardinality(String),
`browser` LowCardinality(String),
`browser_version` LowCardinality(String),
`device` LowCardinality(String),
`brand` LowCardinality(String),
`model` LowCardinality(String),
`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),
INDEX idx_name name TYPE bloom_filter GRANULARITY 1,
INDEX idx_properties_bounce properties['__bounce'] TYPE set(3) GRANULARITY 1,
INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1,
INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (project_id, toDate(created_at), profile_id, name)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS events_bots (
`id` UUID DEFAULT generateUUIDv4(),
`project_id` String,
`name` String,
`type` String,
`path` String,
`created_at` DateTime64(3)
)
ENGINE = MergeTree()
ORDER BY (project_id, created_at)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS profiles (
`id` String CODEC(ZSTD(3)),
`is_external` Bool,
`first_name` String CODEC(ZSTD(3)),
`last_name` String CODEC(ZSTD(3)),
`email` String CODEC(ZSTD(3)),
`avatar` String CODEC(ZSTD(3)),
`properties` Map(String, String) CODEC(ZSTD(3)),
`project_id` String CODEC(ZSTD(3)),
`created_at` DateTime64(3) CODEC(Delta(4), LZ4),
INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
INDEX idx_email email TYPE bloom_filter GRANULARITY 1
)
ENGINE = ReplacingMergeTree(created_at)
PARTITION BY toYYYYMM(created_at)
ORDER BY (project_id, id)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS profile_aliases (
`project_id` String,
`profile_id` String,
`alias` String,
`created_at` DateTime
)
ENGINE = MergeTree()
ORDER BY (project_id, profile_id, alias, created_at)
SETTINGS index_granularity = 8192;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv
ENGINE = AggregatingMergeTree()
PARTITION BY toYYYYMMDD(date)
ORDER BY (project_id, date)
AS SELECT
toDate(created_at) as date,
uniqState(profile_id) as profile_id,
project_id
FROM events
GROUP BY date, project_id;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS cohort_events_mv
ENGINE = AggregatingMergeTree()
ORDER BY (project_id, name, created_at, profile_id)
AS SELECT
project_id,
name,
toDate(created_at) AS created_at,
profile_id,
COUNT() AS event_count
FROM events
WHERE profile_id != device_id
GROUP BY project_id, name, created_at, profile_id;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS distinct_event_names_mv
ENGINE = AggregatingMergeTree()
ORDER BY (project_id, name, created_at)
AS SELECT
project_id,
name,
max(created_at) AS created_at,
count() AS event_count
FROM events
GROUP BY project_id, name;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS event_property_values_mv
ENGINE = AggregatingMergeTree()
ORDER BY (project_id, name, property_key, property_value)
AS SELECT
project_id,
name,
key_value.keys as property_key,
key_value.values as property_value,
created_at
FROM (
SELECT
project_id,
name,
untuple(arrayJoin(properties)) as key_value,
max(created_at) as created_at
FROM events
GROUP BY project_id, name, key_value
)
WHERE property_value != ''
AND property_key != ''
AND property_key NOT IN ('__duration_from', '__properties_from')
GROUP BY project_id, name, property_key, property_value, created_at;

22933
packages/db/code-migrations/4-add-sessions.sql
View File

File diff suppressed because it is too large Load Diff

47
packages/db/code-migrations/5-add-imports-table.sql
View File

@@ -1,47 +0,0 @@
CREATE TABLE IF NOT EXISTS events_imports_replicated ON CLUSTER '{cluster}' (
`id` UUID DEFAULT generateUUIDv4(),
`name` LowCardinality(String),
`sdk_name` LowCardinality(String),
`sdk_version` LowCardinality(String),
`device_id` String CODEC(ZSTD(3)),
`profile_id` String CODEC(ZSTD(3)),
`project_id` String CODEC(ZSTD(3)),
`session_id` String CODEC(LZ4),
`path` String CODEC(ZSTD(3)),
`origin` String CODEC(ZSTD(3)),
`referrer` String CODEC(ZSTD(3)),
`referrer_name` String CODEC(ZSTD(3)),
`referrer_type` LowCardinality(String),
`duration` UInt64 CODEC(Delta(4), LZ4),
`properties` Map(String, String) CODEC(ZSTD(3)),
`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),
`country` LowCardinality(FixedString(2)),
`city` String,
`region` LowCardinality(String),
`longitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`latitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`os` LowCardinality(String),
`os_version` LowCardinality(String),
`browser` LowCardinality(String),
`browser_version` LowCardinality(String),
`device` LowCardinality(String),
`brand` LowCardinality(String),
`model` LowCardinality(String),
`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),
`import_id` String CODEC(ZSTD(3)),
`import_status` LowCardinality(String) DEFAULT 'pending',
`imported_at_meta` DateTime DEFAULT now()
)
ENGINE = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/openpanel/v1/{table}', '{replica}')
PARTITION BY toYYYYMM(imported_at_meta)
ORDER BY (import_id, created_at)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS events_imports ON CLUSTER '{cluster}' AS events_imports_replicated
ENGINE = Distributed('{cluster}', currentDatabase(), events_imports_replicated, cityHash64(import_id));
---
ALTER TABLE events_imports_replicated ON CLUSTER '{cluster}' MODIFY TTL imported_at_meta + INTERVAL 7 DAY;

36
packages/db/code-migrations/6-add-revenue-column.ts Normal file
View File

@@ -0,0 +1,36 @@
import fs from 'node:fs';
import path from 'node:path';
import {
addColumns,
runClickhouseMigrationCommands,
} from '../src/clickhouse/migration';
import { getIsCluster } from './helpers';
export async function up() {
const isClustered = getIsCluster();
const sqls: string[] = [
...addColumns(
'events',
['`revenue` UInt64 AFTER `referrer_type`'],
isClustered,
),
];
fs.writeFileSync(
path.join(__filename.replace('.ts', '.sql')),
sqls
.map((sql) =>
sql
.trim()
.replace(/;$/, '')
.replace(/\n{2,}/g, '\n')
.concat(';'),
)
.join('\n\n---\n\n'),
);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}

2
packages/db/prisma/migrations/20251118100123_add_revenue_tracking_setting_on_project/migration.sql Normal file
View File

@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "public"."projects" ADD COLUMN "allowUnsafeRevenueTracking" BOOLEAN NOT NULL DEFAULT false;

21
packages/db/prisma/schema.prisma
View File

@@ -172,17 +172,18 @@ model Invite {
}
model Project {
id String @id @default(dbgenerated("gen_random_uuid()"))
name String
organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
organizationId String
eventsCount Int @default(0)
types ProjectType[] @default([])
domain String?
cors String[] @default([])
crossDomain Boolean @default(false)
id String @id @default(dbgenerated("gen_random_uuid()"))
name String
organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
organizationId String
eventsCount Int @default(0)
types ProjectType[] @default([])
domain String?
cors String[] @default([])
crossDomain Boolean @default(false)
allowUnsafeRevenueTracking Boolean @default(false)
/// [IPrismaProjectFilters]
filters Json @default("[]")
filters Json @default("[]")
clients Client[]
reports Report[]

38
packages/db/src/buffers/session-buffer.ts
View File

@@ -28,8 +28,24 @@ export class SessionBuffer extends BaseBuffer {
this.redis = getRedisCache();
}
public async getExistingSession(sessionId: string) {
const hit = await this.redis.get(`session:${sessionId}`);
public async getExistingSession(
options:
| {
sessionId: string;
}
| {
projectId: string;
profileId: string;
},
) {
let hit: string | null = null;
if ('sessionId' in options) {
hit = await this.redis.get(`session:${options.sessionId}`);
} else {
hit = await this.redis.get(
`session:${options.projectId}:${options.profileId}`,
);
}
if (hit) {
return getSafeJson<IClickhouseSession>(hit);
@@ -41,7 +57,9 @@ export class SessionBuffer extends BaseBuffer {
async getSession(
event: IClickhouseEvent,
): Promise<[IClickhouseSession] | [IClickhouseSession, IClickhouseSession]> {
const existingSession = await this.getExistingSession(event.session_id);
const existingSession = await this.getExistingSession({
sessionId: event.session_id,
});
if (existingSession) {
const oldSession = assocPath(['sign'], -1, clone(existingSession));
@@ -77,7 +95,9 @@ export class SessionBuffer extends BaseBuffer {
...(event.properties || {}),
...(newSession.properties || {}),
});
// newSession.revenue += event.properties?.__revenue ?? 0;
const addedRevenue = event.name === 'revenue' ? (event.revenue ?? 0) : 0;
newSession.revenue = (newSession.revenue ?? 0) + addedRevenue;
if (event.name === 'screen_view' && event.path) {
newSession.screen_views.push(event.path);
@@ -114,7 +134,7 @@ export class SessionBuffer extends BaseBuffer {
entry_origin: event.origin,
exit_path: event.path,
exit_origin: event.origin,
revenue: 0,
revenue: event.name === 'revenue' ? (event.revenue ?? 0) : 0,
referrer: event.referrer,
referrer_name: event.referrer_name,
referrer_type: event.referrer_type,
@@ -174,6 +194,14 @@ export class SessionBuffer extends BaseBuffer {
'EX',
60 * 60,
);
if (newSession.profile_id) {
multi.set(
`session:${newSession.project_id}:${newSession.profile_id}`,
JSON.stringify(newSession),
'EX',
60 * 60,
);
}
for (const session of sessions) {
multi.rpush(this.redisKey, JSON.stringify(session));
}

16
packages/db/src/clickhouse/migration.ts
View File

@@ -140,10 +140,10 @@ export function addColumns(
isClustered: boolean,
): string[] {
if (isClustered) {
return columns.map(
(col) =>
`ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' ADD COLUMN IF NOT EXISTS ${col}`,
);
return columns.flatMap((col) => [
`ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' ADD COLUMN IF NOT EXISTS ${col}`,
`ALTER TABLE ${tableName} ON CLUSTER '{cluster}' ADD COLUMN IF NOT EXISTS ${col}`,
]);
}
return columns.map(
@@ -160,10 +160,10 @@ export function dropColumns(
isClustered: boolean,
): string[] {
if (isClustered) {
return columnNames.map(
(colName) =>
`ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' DROP COLUMN IF EXISTS ${colName}`,
);
return columnNames.flatMap((colName) => [
`ALTER TABLE ${replicated(tableName)} ON CLUSTER '{cluster}' DROP COLUMN IF EXISTS ${colName}`,
`ALTER TABLE ${tableName} ON CLUSTER '{cluster}' DROP COLUMN IF EXISTS ${colName}`,
]);
}
return columnNames.map(

36
packages/db/src/services/chart.service.ts
View File

@@ -174,23 +174,43 @@ export function getChartSql({
}
if (event.segment === 'property_sum' && event.property) {
sb.select.count = `sum(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
if (event.property === 'revenue') {
sb.select.count = `sum(revenue) as count`;
sb.where.property = `revenue > 0`;
} else {
sb.select.count = `sum(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
}
}
if (event.segment === 'property_average' && event.property) {
sb.select.count = `avg(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
if (event.property === 'revenue') {
sb.select.count = `avg(revenue) as count`;
sb.where.property = `revenue > 0`;
} else {
sb.select.count = `avg(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
}
}
if (event.segment === 'property_max' && event.property) {
sb.select.count = `max(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
if (event.property === 'revenue') {
sb.select.count = `max(revenue) as count`;
sb.where.property = `revenue > 0`;
} else {
sb.select.count = `max(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
}
}
if (event.segment === 'property_min' && event.property) {
sb.select.count = `min(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
if (event.property === 'revenue') {
sb.select.count = `min(revenue) as count`;
sb.where.property = `revenue > 0`;
} else {
sb.select.count = `min(toFloat64(${getSelectPropertyKey(event.property)})) as count`;
sb.where.property = `${getSelectPropertyKey(event.property)} IS NOT NULL AND notEmpty(${getSelectPropertyKey(event.property)})`;
}
}
if (event.segment === 'one_event_per_user') {

54
packages/db/src/services/event.service.ts
View File

@@ -25,6 +25,7 @@ import {
getProfilesCached,
upsertProfile,
} from './profile.service';
import type { IClickhouseSession } from './session.service';
export type IImportedEvent = Omit<
IClickhouseEvent,
@@ -92,12 +93,62 @@ export interface IClickhouseEvent {
imported_at: string | null;
sdk_name: string;
sdk_version: string;
revenue?: number;
// They do not exist here. Just make ts happy for now
profile?: IServiceProfile;
meta?: EventMeta;
}
export function transformSessionToEvent(
session: IClickhouseSession,
): IServiceEvent {
return {
id: '', // Not used
name: 'screen_view',
sessionId: session.id,
profileId: session.profile_id,
path: session.exit_path,
origin: session.exit_origin,
createdAt: convertClickhouseDateToJs(session.ended_at),
referrer: session.referrer,
referrerName: session.referrer_name,
referrerType: session.referrer_type,
os: session.os,
osVersion: session.os_version,
browser: session.browser,
browserVersion: session.browser_version,
device: session.device,
brand: session.brand,
model: session.model,
country: session.country,
region: session.region,
city: session.city,
longitude: session.longitude,
latitude: session.latitude,
projectId: session.project_id,
deviceId: session.device_id,
duration: 0,
revenue: session.revenue,
properties: {
...session.properties,
is_bounce: session.is_bounce,
__query: {
utm_medium: session.utm_medium,
utm_source: session.utm_source,
utm_campaign: session.utm_campaign,
utm_content: session.utm_content,
utm_term: session.utm_term,
},
},
profile: undefined,
meta: undefined,
importedAt: undefined,
sdkName: undefined,
sdkVersion: undefined,
};
}
export function transformEvent(event: IClickhouseEvent): IServiceEvent {
return {
id: event.id,
@@ -131,6 +182,7 @@ export function transformEvent(event: IClickhouseEvent): IServiceEvent {
sdkName: event.sdk_name,
sdkVersion: event.sdk_version,
profile: event.profile,
revenue: event.revenue,
};
}
@@ -178,6 +230,7 @@ export interface IServiceEvent {
meta: EventMeta | undefined;
sdkName: string | undefined;
sdkVersion: string | undefined;
revenue?: number;
}
type SelectHelper<T> = {
@@ -336,6 +389,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
imported_at: null,
sdk_name: payload.sdkName ?? '',
sdk_version: payload.sdkVersion ?? '',
revenue: payload.revenue,
};
const promises = [sessionBuffer.add(event), eventBuffer.add(event)];

33
packages/db/src/services/overview.service.ts
View File

@@ -104,6 +104,7 @@ export class OverviewService {
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
total_revenue: number;
};
series: {
date: string;
@@ -113,6 +114,7 @@ export class OverviewService {
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
total_revenue: number;
}[];
}> {
const where = this.getRawWhereClause('sessions', filters);
@@ -122,6 +124,7 @@ export class OverviewService {
.select([
`${clix.toStartOf('created_at', interval, timezone)} AS date`,
'round((countIf(is_bounce = 1 AND sign = 1) * 100.) / countIf(sign = 1), 2) AS bounce_rate',
'sum(revenue * sign) AS total_revenue',
])
.from(TABLE_NAMES.sessions, true)
.where('sign', '=', 1)
@@ -165,10 +168,17 @@ export class OverviewService {
.from('session_agg')
.where('date', '=', rollupDate),
)
.with(
'overall_total_revenue',
clix(this.client, timezone)
.select(['total_revenue'])
.from('session_agg')
.where('date', '=', rollupDate),
)
.with(
'daily_stats',
clix(this.client, timezone)
.select(['date', 'bounce_rate'])
.select(['date', 'bounce_rate', 'total_revenue'])
.from('session_agg')
.where('date', '!=', rollupDate),
)
@@ -181,9 +191,11 @@ export class OverviewService {
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
total_revenue: number;
overall_unique_visitors: number;
overall_total_sessions: number;
overall_bounce_rate: number;
overall_total_revenue: number;
}>([
`${clix.toStartOf('e.created_at', interval)} AS date`,
'ds.bounce_rate as bounce_rate',
@@ -193,9 +205,11 @@ export class OverviewService {
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
'count(*) AS total_screen_views',
'round((count(*) * 1.) / uniq(e.session_id), 2) AS views_per_session',
'ds.total_revenue AS total_revenue',
'(SELECT unique_visitors FROM overall_unique_visitors) AS overall_unique_visitors',
'(SELECT total_sessions FROM overall_unique_visitors) AS overall_total_sessions',
'(SELECT bounce_rate FROM overall_bounce_rate) AS overall_bounce_rate',
'(SELECT total_revenue FROM overall_total_revenue) AS overall_total_revenue',
])
.from(`${TABLE_NAMES.events} AS e`)
.leftJoin(
@@ -209,7 +223,7 @@ export class OverviewService {
clix.datetime(endDate, 'toDateTime'),
])
.rawWhere(this.getRawWhereClause('events', filters))
.groupBy(['date', 'ds.bounce_rate'])
.groupBy(['date', 'ds.bounce_rate', 'ds.total_revenue'])
.orderBy('date', 'ASC')
.fill(
clix.toStartOf(
@@ -234,7 +248,8 @@ export class OverviewService {
(item) =>
item.overall_bounce_rate !== null ||
item.overall_total_sessions !== null ||
item.overall_unique_visitors !== null,
item.overall_unique_visitors !== null ||
item.overall_total_revenue !== null,
);
return {
metrics: {
@@ -250,12 +265,14 @@ export class OverviewService {
views_per_session: average(
res.map((item) => item.views_per_session),
),
total_revenue: anyRowWithData?.overall_total_revenue ?? 0,
},
series: res.map(
omit([
'overall_bounce_rate',
'overall_unique_visitors',
'overall_total_sessions',
'overall_total_revenue',
]),
),
};
@@ -271,6 +288,7 @@ export class OverviewService {
avg_session_duration: number;
total_screen_views: number;
views_per_session: number;
total_revenue: number;
}>([
`${clix.toStartOf('created_at', interval, timezone)} AS date`,
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
@@ -280,6 +298,7 @@ export class OverviewService {
'if(isNaN(_avg_session_duration), 0, _avg_session_duration) AS avg_session_duration',
'sum(sign * screen_view_count) AS total_screen_views',
'round(sum(sign * screen_view_count) * 1.0 / sum(sign), 2) AS views_per_session',
'sum(revenue * sign) AS total_revenue',
])
.from('sessions')
.where('created_at', 'BETWEEN', [
@@ -320,6 +339,7 @@ export class OverviewService {
avg_session_duration: res[0]?.avg_session_duration ?? 0,
total_screen_views: res[0]?.total_screen_views ?? 0,
views_per_session: res[0]?.views_per_session ?? 0,
total_revenue: res[0]?.total_revenue ?? 0,
},
series: res
.slice(1)
@@ -394,6 +414,7 @@ export class OverviewService {
'entry_path',
'entry_origin',
'coalesce(round(countIf(is_bounce = 1 AND sign = 1) * 100.0 / countIf(sign = 1), 2), 0) as bounce_rate',
'sum(revenue * sign) as revenue',
])
.from(TABLE_NAMES.sessions, true)
.where('sign', '=', 1)
@@ -417,6 +438,7 @@ export class OverviewService {
avg_duration: number;
bounce_rate: number;
sessions: number;
revenue: number;
}>([
'p.title',
'p.origin',
@@ -424,6 +446,7 @@ export class OverviewService {
'p.avg_duration',
'p.count as sessions',
'b.bounce_rate',
'coalesce(b.revenue, 0) as revenue',
])
.from('page_stats p', false)
.leftJoin(
@@ -465,12 +488,14 @@ export class OverviewService {
avg_duration: number;
bounce_rate: number;
sessions: number;
revenue: number;
}>([
`${mode}_origin AS origin`,
`${mode}_path AS path`,
'round(avg(duration * sign)/1000, 2) as avg_duration',
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) as bounce_rate',
'sum(sign) as sessions',
'sum(revenue * sign) as revenue',
])
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)
@@ -566,12 +591,14 @@ export class OverviewService {
sessions: number;
bounce_rate: number;
avg_session_duration: number;
revenue: number;
}>([
prefixColumn && `${prefixColumn} as prefix`,
`nullIf(${column}, '') as name`,
'sum(sign) as sessions',
'round(sum(sign * is_bounce) * 100.0 / sum(sign), 2) AS bounce_rate',
'round(avgIf(duration, duration > 0 AND sign > 0), 2)/1000 AS avg_session_duration',
'sum(revenue * sign) as revenue',
])
.from(TABLE_NAMES.sessions, true)
.where('project_id', '=', projectId)

7
packages/db/src/services/profile.service.ts
View File

@@ -28,6 +28,7 @@ export type IProfileMetrics = {
avgEventsPerSession: number;
conversionEvents: number;
avgTimeBetweenSessions: number;
revenue: number;
};
export function getProfileMetrics(profileId: string, projectId: string) {
return chQuery<
@@ -76,6 +77,9 @@ export function getProfileMetrics(profileId: string, projectId: string) {
WHEN (SELECT sessions FROM sessions) <= 1 THEN 0
ELSE round(dateDiff('second', (SELECT firstSeen FROM firstSeen), (SELECT lastSeen FROM lastSeen)) / nullIf((SELECT sessions FROM sessions) - 1, 0), 1)
END as avgTimeBetweenSessions
),
revenue AS (
SELECT sum(revenue) as revenue FROM ${TABLE_NAMES.events} WHERE name = 'revenue' AND profile_id = ${sqlstring.escape(profileId)} AND project_id = ${sqlstring.escape(projectId)}
)
SELECT
(SELECT lastSeen FROM lastSeen) as lastSeen,
@@ -89,7 +93,8 @@ export function getProfileMetrics(profileId: string, projectId: string) {
(SELECT bounceRate FROM bounceRate) as bounceRate,
(SELECT avgEventsPerSession FROM avgEventsPerSession) as avgEventsPerSession,
(SELECT conversionEvents FROM conversionEvents) as conversionEvents,
(SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions
(SELECT avgTimeBetweenSessions FROM avgTimeBetweenSessions) as avgTimeBetweenSessions,
(SELECT revenue FROM revenue) as revenue
`)
.then((data) => data[0]!)
.then((data) => {

19
packages/db/src/session-consistency.ts
View File

@@ -209,12 +209,27 @@ export function sessionConsistency() {
// Since the check probably goes to the primary anyways it will always be true,
// Not sure how to check LSN on the actual replica that will be used for the read.
if (
model !== 'Session' &&
isReadOperation(operation) &&
sessionId &&
(await getCachedWalLsn(sessionId))
) {
// This will force readReplicas extension to use primary
__internalParams.transaction = true;
const MAX_RETRIES = 3;
const INITIAL_RETRY_DELAY_MS = 50;
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
const result = await query(args);
if (result !== null) {
return result;
}
// If not the last attempt, wait with exponential backoff before retrying
if (attempt < MAX_RETRIES - 1) {
const delayMs = INITIAL_RETRY_DELAY_MS * 2 ** attempt;
await sleep(delayMs);
}
}
}
return query(args);

22
packages/sdks/sdk/src/index.ts
View File

@@ -183,6 +183,28 @@ export class OpenPanel {
});
}
async revenue(
amount: number,
properties?: TrackProperties & { deviceId?: string },
) {
const deviceId = properties?.deviceId;
delete properties?.deviceId;
return this.track('revenue', {
...(properties ?? {}),
...(deviceId ? { __deviceId: deviceId } : {}),
__revenue: amount,
});
}
async fetchDeviceId(): Promise<string> {
const result = await this.api.fetch<undefined, { deviceId: string }>(
'/track/device-id',
undefined,
{ method: 'GET', keepalive: false },
);
return result?.deviceId ?? '';
}
clear() {
this.profileId = undefined;
// should we force a session end here?

47
packages/sdks/web/src/index.ts
View File

@@ -20,9 +20,15 @@ function toCamelCase(str: string) {
);
}
type PendingRevenue = {
amount: number;
properties?: Record<string, unknown>;
};
export class OpenPanel extends OpenPanelBase {
private lastPath = '';
private debounceTimer: any;
private pendingRevenues: PendingRevenue[] = [];
constructor(public options: OpenPanelOptions) {
super({
@@ -34,6 +40,18 @@ export class OpenPanel extends OpenPanelBase {
if (!this.isServer()) {
console.log('OpenPanel.dev - Initialized', this.options);
try {
const pending = sessionStorage.getItem('openpanel-pending-revenues');
if (pending) {
const parsed = JSON.parse(pending);
if (Array.isArray(parsed)) {
this.pendingRevenues = parsed;
}
}
} catch {
this.pendingRevenues = [];
}
this.setGlobalProperties({
__referrer: document.referrer,
});
@@ -191,4 +209,33 @@ export class OpenPanel extends OpenPanelBase {
__title: document.title,
});
}
async flushRevenue() {
const promises = this.pendingRevenues.map((pending) =>
super.revenue(pending.amount, pending.properties),
);
await Promise.all(promises);
this.clearRevenue();
}
clearRevenue() {
this.pendingRevenues = [];
if (!this.isServer()) {
try {
sessionStorage.removeItem('openpanel-pending-revenues');
} catch {}
}
}
pendingRevenue(amount: number, properties?: Record<string, unknown>) {
this.pendingRevenues.push({ amount, properties });
if (!this.isServer()) {
try {
sessionStorage.setItem(
'openpanel-pending-revenues',
JSON.stringify(this.pendingRevenues),
);
} catch {}
}
}
}

6
packages/sdks/web/src/types.d.ts vendored
View File

@@ -8,7 +8,11 @@ type ExposedMethodsNames =
| 'alias'
| 'increment'
| 'decrement'
| 'clear';
| 'clear'
| 'revenue'
| 'flushRevenue'
| 'clearRevenue'
| 'pendingRevenue';
export type ExposedMethods = {
[K in ExposedMethodsNames]: OpenPanel[K] extends (...args: any[]) => any

17
packages/trpc/src/routers/chart.ts
View File

@@ -60,13 +60,18 @@ export const chartRouter = createTRPCRouter({
}),
)
.query(async ({ input: { projectId } }) => {
const chartPromise = chQuery<{ value: number; date: Date }>(
const { timezone } = await getSettingsForProject(projectId);
const chartPromise = chQuery<{
value: number;
date: Date;
revenue: number;
}>(
`SELECT
uniqHLL12(profile_id) as value,
toStartOfDay(created_at) as date
toStartOfDay(created_at) as date,
sum(revenue * sign) as revenue
FROM ${TABLE_NAMES.sessions}
WHERE
sign = 1 AND
project_id = ${sqlstring.escape(projectId)} AND
created_at >= now() - interval '3 month'
GROUP BY date
@@ -74,22 +79,25 @@ export const chartRouter = createTRPCRouter({
WITH FILL FROM toStartOfDay(now() - interval '1 month')
TO toStartOfDay(now())
STEP INTERVAL 1 day
SETTINGS session_timezone = '${timezone}'
`,
);
const metricsPromise = clix(ch)
const metricsPromise = clix(ch, timezone)
.select<{
months_3: number;
months_3_prev: number;
month: number;
day: number;
day_prev: number;
revenue: number;
}>([
'uniqHLL12(if(created_at >= (now() - toIntervalMonth(3)), profile_id, null)) AS months_3',
'uniqHLL12(if(created_at >= (now() - toIntervalMonth(6)) AND created_at < (now() - toIntervalMonth(3)), profile_id, null)) AS months_3_prev',
'uniqHLL12(if(created_at >= (now() - toIntervalMonth(1)), profile_id, null)) AS month',
'uniqHLL12(if(created_at >= (now() - toIntervalDay(1)), profile_id, null)) AS day',
'uniqHLL12(if(created_at >= (now() - toIntervalDay(2)) AND created_at < (now() - toIntervalDay(1)), profile_id, null)) AS day_prev',
'sum(revenue * sign) as revenue',
])
.from(TABLE_NAMES.sessions)
.where('project_id', '=', projectId)
@@ -207,6 +215,7 @@ export const chartRouter = createTRPCRouter({
}
properties.push(
'revenue',
'has_profile',
'path',
'origin',

10
packages/trpc/src/routers/overview.ts
View File

@@ -103,7 +103,6 @@ export const overviewRouter = createTRPCRouter({
])
.from(TABLE_NAMES.events)
.where('project_id', '=', input.projectId)
.where('name', '=', 'session_start')
.where('created_at', '>=', clix.exp('now() - INTERVAL 30 MINUTE'));
// Get counts per minute for the last 30 minutes
@@ -119,7 +118,6 @@ export const overviewRouter = createTRPCRouter({
])
.from(TABLE_NAMES.events)
.where('project_id', '=', input.projectId)
.where('name', 'IN', ['session_start', 'screen_view'])
.where('created_at', '>=', clix.exp('now() - INTERVAL 30 MINUTE'))
.groupBy(['minute'])
.orderBy('minute', 'ASC')
@@ -138,11 +136,10 @@ export const overviewRouter = createTRPCRouter({
}>([
`${clix.toStartOf('created_at', 'minute')} as minute`,
'referrer_name',
'count(*) as count',
'uniq(session_id) as count',
])
.from(TABLE_NAMES.events)
.where('project_id', '=', input.projectId)
.where('name', '=', 'session_start')
.where('created_at', '>=', clix.exp('now() - INTERVAL 30 MINUTE'))
.where('referrer_name', '!=', '')
.where('referrer_name', 'IS NOT NULL')
@@ -154,11 +151,10 @@ export const overviewRouter = createTRPCRouter({
const referrersQuery = clix(ch, timezone)
.select<{ referrer: string; count: number }>([
'referrer_name as referrer',
'count(*) as count',
'uniq(session_id) as count',
])
.from(TABLE_NAMES.events)
.where('project_id', '=', input.projectId)
.where('name', '=', 'session_start')
.where('created_at', '>=', clix.exp('now() - INTERVAL 30 MINUTE'))
.where('referrer_name', '!=', '')
.where('referrer_name', 'IS NOT NULL')
@@ -234,6 +230,7 @@ export const overviewRouter = createTRPCRouter({
previous?.metrics.avg_session_duration || null,
prev_views_per_session: previous?.metrics.views_per_session || null,
prev_total_sessions: previous?.metrics.total_sessions || null,
prev_total_revenue: previous?.metrics.total_revenue || null,
},
series: current.series.map((item, index) => {
const prev = previous?.series[index];
@@ -246,6 +243,7 @@ export const overviewRouter = createTRPCRouter({
prev_avg_session_duration: prev?.avg_session_duration,
prev_views_per_session: prev?.views_per_session,
prev_total_sessions: prev?.total_sessions,
prev_total_revenue: prev?.total_revenue,
};
}),
};

2
packages/trpc/src/routers/project.ts
View File

@@ -84,6 +84,7 @@ export const projectRouter = createTRPCRouter({
input.cors === undefined
? undefined
: input.cors.map((c) => stripTrailingSlash(c)) || [],
allowUnsafeRevenueTracking: input.allowUnsafeRevenueTracking,
},
include: {
clients: {
@@ -123,6 +124,7 @@ export const projectRouter = createTRPCRouter({
domain: input.domain,
cors: input.cors,
crossDomain: false,
allowUnsafeRevenueTracking: false,
filters: [],
clients: {
create: data,

2
packages/trpc/src/trpc.ts
View File

@@ -191,7 +191,7 @@ export const cacheMiddleware = (
key += JSON.stringify(rawInput).replace(/\"/g, "'");
}
const cache = await getRedisCache().getJson(key);
if (cache) {
if (cache && process.env.NODE_ENV === 'production') {
return {
ok: true,
data: cache,

1
packages/validation/src/index.ts
View File

@@ -379,6 +379,7 @@ export const zProject = z.object({
domain: z.string().url().or(z.literal('').or(z.null())),
cors: z.array(z.string()).default([]),
crossDomain: z.boolean().default(false),
allowUnsafeRevenueTracking: z.boolean().default(false),
});
export type IProjectEdit = z.infer<typeof zProject>;