zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: revenue tracking

Browse Source 
* wip

* wip

* wip

* wip

* show revenue better on overview

* align realtime and overview counters

* update revenue docs

* always return device id

* add project settings, improve projects charts,

* fix: comments

* fixes

* fix migration

* ignore sql files

* fix comments
This commit is contained in:
Carl-Gerhard Lindesvärd
2025-11-19 14:27:34 +01:00
committed by GitHub
parent d61cbf6f2c
commit 790801b728
58 changed files with 2191 additions and 23691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
packages/db/code-migrations/3-init-ch.sql
View File

@@ -1,167 +0,0 @@
CREATE DATABASE IF NOT EXISTS openpanel;
---
CREATE TABLE IF NOT EXISTS self_hosting (
`created_at` Date,
`domain` String,
`count` UInt64
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (domain, created_at);
---
CREATE TABLE IF NOT EXISTS events (
`id` UUID DEFAULT generateUUIDv4(),
`name` LowCardinality(String),
`sdk_name` LowCardinality(String),
`sdk_version` LowCardinality(String),
`device_id` String CODEC(ZSTD(3)),
`profile_id` String CODEC(ZSTD(3)),
`project_id` String CODEC(ZSTD(3)),
`session_id` String CODEC(LZ4),
`path` String CODEC(ZSTD(3)),
`origin` String CODEC(ZSTD(3)),
`referrer` String CODEC(ZSTD(3)),
`referrer_name` String CODEC(ZSTD(3)),
`referrer_type` LowCardinality(String),
`duration` UInt64 CODEC(Delta(4), LZ4),
`properties` Map(String, String) CODEC(ZSTD(3)),
`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),
`country` LowCardinality(FixedString(2)),
`city` String,
`region` LowCardinality(String),
`longitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`latitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`os` LowCardinality(String),
`os_version` LowCardinality(String),
`browser` LowCardinality(String),
`browser_version` LowCardinality(String),
`device` LowCardinality(String),
`brand` LowCardinality(String),
`model` LowCardinality(String),
`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),
INDEX idx_name name TYPE bloom_filter GRANULARITY 1,
INDEX idx_properties_bounce properties['__bounce'] TYPE set(3) GRANULARITY 1,
INDEX idx_origin origin TYPE bloom_filter(0.05) GRANULARITY 1,
INDEX idx_path path TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (project_id, toDate(created_at), profile_id, name)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS events_bots (
`id` UUID DEFAULT generateUUIDv4(),
`project_id` String,
`name` String,
`type` String,
`path` String,
`created_at` DateTime64(3)
)
ENGINE = MergeTree()
ORDER BY (project_id, created_at)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS profiles (
`id` String CODEC(ZSTD(3)),
`is_external` Bool,
`first_name` String CODEC(ZSTD(3)),
`last_name` String CODEC(ZSTD(3)),
`email` String CODEC(ZSTD(3)),
`avatar` String CODEC(ZSTD(3)),
`properties` Map(String, String) CODEC(ZSTD(3)),
`project_id` String CODEC(ZSTD(3)),
`created_at` DateTime64(3) CODEC(Delta(4), LZ4),
INDEX idx_first_name first_name TYPE bloom_filter GRANULARITY 1,
INDEX idx_last_name last_name TYPE bloom_filter GRANULARITY 1,
INDEX idx_email email TYPE bloom_filter GRANULARITY 1
)
ENGINE = ReplacingMergeTree(created_at)
PARTITION BY toYYYYMM(created_at)
ORDER BY (project_id, id)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS profile_aliases (
`project_id` String,
`profile_id` String,
`alias` String,
`created_at` DateTime
)
ENGINE = MergeTree()
ORDER BY (project_id, profile_id, alias, created_at)
SETTINGS index_granularity = 8192;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv
ENGINE = AggregatingMergeTree()
PARTITION BY toYYYYMMDD(date)
ORDER BY (project_id, date)
AS SELECT
toDate(created_at) as date,
uniqState(profile_id) as profile_id,
project_id
FROM events
GROUP BY date, project_id;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS cohort_events_mv
ENGINE = AggregatingMergeTree()
ORDER BY (project_id, name, created_at, profile_id)
AS SELECT
project_id,
name,
toDate(created_at) AS created_at,
profile_id,
COUNT() AS event_count
FROM events
WHERE profile_id != device_id
GROUP BY project_id, name, created_at, profile_id;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS distinct_event_names_mv
ENGINE = AggregatingMergeTree()
ORDER BY (project_id, name, created_at)
AS SELECT
project_id,
name,
max(created_at) AS created_at,
count() AS event_count
FROM events
GROUP BY project_id, name;
---
CREATE MATERIALIZED VIEW IF NOT EXISTS event_property_values_mv
ENGINE = AggregatingMergeTree()
ORDER BY (project_id, name, property_key, property_value)
AS SELECT
project_id,
name,
key_value.keys as property_key,
key_value.values as property_value,
created_at
FROM (
SELECT
project_id,
name,
untuple(arrayJoin(properties)) as key_value,
max(created_at) as created_at
FROM events
GROUP BY project_id, name, key_value
)
WHERE property_value != ''
AND property_key != ''
AND property_key NOT IN ('__duration_from', '__properties_from')
GROUP BY project_id, name, property_key, property_value, created_at;

22933
packages/db/code-migrations/4-add-sessions.sql
View File

File diff suppressed because it is too large Load Diff

47
packages/db/code-migrations/5-add-imports-table.sql
View File

@@ -1,47 +0,0 @@
CREATE TABLE IF NOT EXISTS events_imports_replicated ON CLUSTER '{cluster}' (
`id` UUID DEFAULT generateUUIDv4(),
`name` LowCardinality(String),
`sdk_name` LowCardinality(String),
`sdk_version` LowCardinality(String),
`device_id` String CODEC(ZSTD(3)),
`profile_id` String CODEC(ZSTD(3)),
`project_id` String CODEC(ZSTD(3)),
`session_id` String CODEC(LZ4),
`path` String CODEC(ZSTD(3)),
`origin` String CODEC(ZSTD(3)),
`referrer` String CODEC(ZSTD(3)),
`referrer_name` String CODEC(ZSTD(3)),
`referrer_type` LowCardinality(String),
`duration` UInt64 CODEC(Delta(4), LZ4),
`properties` Map(String, String) CODEC(ZSTD(3)),
`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3)),
`country` LowCardinality(FixedString(2)),
`city` String,
`region` LowCardinality(String),
`longitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`latitude` Nullable(Float32) CODEC(Gorilla, LZ4),
`os` LowCardinality(String),
`os_version` LowCardinality(String),
`browser` LowCardinality(String),
`browser_version` LowCardinality(String),
`device` LowCardinality(String),
`brand` LowCardinality(String),
`model` LowCardinality(String),
`imported_at` Nullable(DateTime) CODEC(Delta(4), LZ4),
`import_id` String CODEC(ZSTD(3)),
`import_status` LowCardinality(String) DEFAULT 'pending',
`imported_at_meta` DateTime DEFAULT now()
)
ENGINE = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/openpanel/v1/{table}', '{replica}')
PARTITION BY toYYYYMM(imported_at_meta)
ORDER BY (import_id, created_at)
SETTINGS index_granularity = 8192;
---
CREATE TABLE IF NOT EXISTS events_imports ON CLUSTER '{cluster}' AS events_imports_replicated
ENGINE = Distributed('{cluster}', currentDatabase(), events_imports_replicated, cityHash64(import_id));
---
ALTER TABLE events_imports_replicated ON CLUSTER '{cluster}' MODIFY TTL imported_at_meta + INTERVAL 7 DAY;

36
packages/db/code-migrations/6-add-revenue-column.ts Normal file
View File

@@ -0,0 +1,36 @@
import fs from 'node:fs';
import path from 'node:path';
import {
addColumns,
runClickhouseMigrationCommands,
} from '../src/clickhouse/migration';
import { getIsCluster } from './helpers';
export async function up() {
const isClustered = getIsCluster();
const sqls: string[] = [
...addColumns(
'events',
['`revenue` UInt64 AFTER `referrer_type`'],
isClustered,
),
];
fs.writeFileSync(
path.join(__filename.replace('.ts', '.sql')),
sqls
.map((sql) =>
sql
.trim()
.replace(/;$/, '')
.replace(/\n{2,}/g, '\n')
.concat(';'),
)
.join('\n\n---\n\n'),
);
if (!process.argv.includes('--dry')) {
await runClickhouseMigrationCommands(sqls);
}
}