zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

first working cli importer

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-07-21 23:42:00 +02:00
committed by Carl-Gerhard Lindesvärd
parent bf0c14cc88
commit 1b613538cc
23 changed files with 403 additions and 920 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
packages/db/clickhouse_init.sql
View File

@@ -1,39 +1,5 @@
CREATE DATABASE IF NOT EXISTS openpanel;
CREATE TABLE IF NOT EXISTS openpanel.events (
`id` UUID DEFAULT generateUUIDv4(),
`name` String,
`device_id` String,
`profile_id` String,
`project_id` String,
`session_id` String,
`path` String,
`origin` String,
`referrer` String,
`referrer_name` String,
`referrer_type` String,
`duration` UInt64,
`properties` Map(String, String),
`created_at` DateTime64(3),
`country` String,
`city` String,
`region` String,
`longitude` Nullable(Float32),
`latitude` Nullable(Float32),
`os` String,
`os_version` String,
`browser` String,
`browser_version` String,
-- device: mobile/desktop/tablet
`device` String,
-- brand: (Samsung, OnePlus)
`brand` String,
-- model: (Samsung Galaxy, iPhone X)
`model` String
) ENGINE MergeTree
ORDER BY
(project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS openpanel.events_v2 (
`id` UUID DEFAULT generateUUIDv4(),
`name` String,
@@ -58,26 +24,17 @@ CREATE TABLE IF NOT EXISTS openpanel.events_v2 (
`os_version` String,
`browser` String,
`browser_version` String,
-- device: mobile/desktop/tablet
`device` String,
-- brand: (Samsung, OnePlus)
`brand` String,
-- model: (Samsung Galaxy, iPhone X)
`model` String
) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at)
`model` String,
`imported_at` Nullable(DateTime),
INDEX idx_name name TYPE bloom_filter GRANULARITY 1,
INDEX idx_properties_bounce properties ['__bounce'] TYPE
set
(3) GRANULARITY 1
) ENGINE = MergeTree PARTITION BY toYYYYMM(created_at)
ORDER BY
(project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
ALTER TABLE
events DROP COLUMN utm_source,
DROP COLUMN utm_medium,
DROP COLUMN utm_campaign,
DROP COLUMN utm_term,
DROP COLUMN utm_content,
DROP COLUMN sdk,
DROP COLUMN sdk_version,
DROP COLUMN client_type,
DROP COLUMN continent;
(project_id, toDate(created_at), profile_id, name) SETTINGS index_granularity = 8192;
CREATE TABLE IF NOT EXISTS openpanel.events_bots (
`id` UUID DEFAULT generateUUIDv4(),

123
packages/db/clickhouse_tables.sql
View File

@@ -1,123 +0,0 @@
CREATE TABLE openpanel.events (
`id` UUID DEFAULT generateUUIDv4(),
`name` String,
`device_id` String,
`profile_id` String,
`project_id` String,
`session_id` String,
`path` String,
`origin` String,
`referrer` String,
`referrer_name` String,
`referrer_type` String,
`duration` UInt64,
`properties` Map(String, String),
`created_at` DateTime64(3),
`country` String,
`city` String,
`region` String,
`longitude` Int16,
`latitude` Int16,
`os` String,
`os_version` String,
`browser` String,
`browser_version` String,
-- device: mobile/desktop/tablet
`device` String,
-- brand: (Samsung, OnePlus)
`brand` String,
-- model: (Samsung Galaxy, iPhone X)
`model` String
) ENGINE MergeTree
ORDER BY
(project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
CREATE TABLE openpanel.events_bots (
`id` UUID DEFAULT generateUUIDv4(),
`project_id` String,
`name` String,
`type` String,
`path` String,
`created_at` DateTime64(3),
) ENGINE MergeTree
ORDER BY
(project_id, created_at) SETTINGS index_granularity = 8192;
CREATE TABLE openpanel.profiles (
`id` String,
`first_name` String,
`last_name` String,
`email` String,
`avatar` String,
`properties` Map(String, String),
`project_id` String,
`created_at` DateTime
) ENGINE = ReplacingMergeTree(created_at)
ORDER BY
(id) SETTINGS index_granularity = 8192;
ALTER TABLE
events
ADD
COLUMN origin String
AFTER
path;
ALTER TABLE
events DROP COLUMN id;
CREATE TABLE ba (
`id` UUID DEFAULT generateUUIDv4(),
`a` String,
`b` String
) ENGINE MergeTree
ORDER BY
(a, b) SETTINGS index_granularity = 8192;
ALTER TABLE
events_bots
ADD
COLUMN id UUID DEFAULT generateUUIDv4() FIRST;
ALTER TABLE
events
ADD
COLUMN longitude Nullable(Float32);
ALTER TABLE
events
ADD
COLUMN latitude Nullable(Float32);
--- Materialized views (DAU)
CREATE MATERIALIZED VIEW dau_mv ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMMDD(date)
ORDER BY
(project_id, date) POPULATE AS
SELECT
toDate(created_at) as date,
uniqState(profile_id) as profile_id,
project_id
FROM
events
GROUP BY
date,
project_id;
-- DROP external_id and add is_external column
ALTER TABLE
profiles DROP COLUMN external_id;
ALTER TABLE
profiles
ADD
COLUMN is_external Boolean
AFTER
id;
ALTER TABLE
profiles
UPDATE
is_external = length(id) != 32
WHERE
true
and length(id) != 32;

6
packages/db/src/clickhouse-client.ts
View File

@@ -11,8 +11,8 @@ export const originalCh = createClient({
username: process.env.CLICKHOUSE_USER,
password: process.env.CLICKHOUSE_PASSWORD,
database: process.env.CLICKHOUSE_DB,
max_open_connections: 10,
request_timeout: 10000,
max_open_connections: 30,
request_timeout: 30000,
keep_alive: {
enabled: true,
idle_socket_ttl: 8000,
@@ -92,7 +92,7 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
};
console.log(
`Query: (${Date.now() - start}ms, ${response.statistics?.elapsed}ms)`,
`Query: (${Date.now() - start}ms, ${response.statistics?.elapsed}ms), Rows: ${json.rows}`,
query
);

58
packages/db/src/services/chart.service.ts
View File

@@ -125,33 +125,67 @@ export function getEventFiltersWhereClause(filters: IChartEventFilter[]) {
}
if (name.startsWith('properties.')) {
const propertyKey = name
.replace(/^properties\./, '')
.replace('.*.', '.%.');
const isWildcard = propertyKey.includes('%');
const whereFrom = `arrayMap(x -> trim(x), mapValues(mapExtractKeyLike(properties, ${escape(
name.replace(/^properties\./, '').replace('.*.', '.%.')
)})))`;
switch (operator) {
case 'is': {
where[id] = `arrayExists(x -> ${value
.map((val) => `x = ${escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x = ${escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `properties['${propertyKey}'] IN (${value
.map((val) => escape(String(val).trim()))
.join(', ')})`;
}
break;
}
case 'isNot': {
where[id] = `arrayExists(x -> ${value
.map((val) => `x != ${escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x != ${escape(String(val).trim())}`)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = `properties['${propertyKey}'] NOT IN (${value
.map((val) => escape(String(val).trim()))
.join(', ')})`;
}
break;
}
case 'contains': {
where[id] = `arrayExists(x -> ${value
.map((val) => `x LIKE ${escape(`%${String(val).trim()}%`)}`)
.join(' OR ')}, ${whereFrom})`;
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x LIKE ${escape(`%${String(val).trim()}%`)}`)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = value
.map(
(val) =>
`properties['${propertyKey}'] LIKE ${escape(`%${String(val).trim()}%`)}`
)
.join(' OR ');
}
break;
}
case 'doesNotContain': {
where[id] = `arrayExists(x -> ${value
.map((val) => `x NOT LIKE ${escape(`%${String(val).trim()}%`)}`)
.join(' OR ')}, ${whereFrom})`;
if (isWildcard) {
where[id] = `arrayExists(x -> ${value
.map((val) => `x NOT LIKE ${escape(`%${String(val).trim()}%`)}`)
.join(' OR ')}, ${whereFrom})`;
} else {
where[id] = value
.map(
(val) =>
`properties['${propertyKey}'] NOT LIKE ${escape(`%${String(val).trim()}%`)}`
)
.join(' OR ');
}
break;
}
}

18
packages/db/src/services/event.service.ts
View File

@@ -21,6 +21,13 @@ import { getEventFiltersWhereClause } from './chart.service';
import { getProfiles, upsertProfile } from './profile.service';
import type { IServiceProfile } from './profile.service';
export type IImportedEvent = Omit<
IClickhouseEvent,
'properties' | 'profile' | 'meta' | 'imported_at'
> & {
properties: Record<string, unknown>;
};
export interface IClickhouseEvent {
id: string;
name: string;
@@ -34,7 +41,7 @@ export interface IClickhouseEvent {
referrer_name: string;
referrer_type: string;
duration: number;
properties: Record<string, string | number | boolean>;
properties: Record<string, string | number | boolean | undefined | null>;
created_at: string;
country: string;
city: string;
@@ -48,6 +55,7 @@ export interface IClickhouseEvent {
device: string;
brand: string;
model: string;
imported_at: string | null;
// They do not exist here. Just make ts happy for now
profile?: IServiceProfile;
@@ -86,6 +94,7 @@ export function transformEvent(
referrerType: event.referrer_type,
profile: event.profile,
meta: event.meta,
importedAt: event.imported_at ? new Date(event.imported_at) : null,
};
}
@@ -119,6 +128,7 @@ export interface IServiceCreateEventPayload {
referrer: string | undefined;
referrerName: string | undefined;
referrerType: string | undefined;
importedAt: Date | null;
profile: IServiceProfile | undefined;
meta: EventMeta | undefined;
}
@@ -221,7 +231,10 @@ export async function getEvents(
}
export async function createEvent(
payload: Omit<IServiceCreateEventPayload, 'id'>
payload: Omit<
IServiceCreateEventPayload,
'id' | 'importedAt' | 'profile' | 'meta'
>
) {
if (!payload.profileId) {
payload.profileId = payload.deviceId;
@@ -283,6 +296,7 @@ export async function createEvent(
referrer: payload.referrer ?? '',
referrer_name: payload.referrerName ?? '',
referrer_type: payload.referrerType ?? '',
imported_at: null,
};
await eventBuffer.insert(event);

9
packages/db/src/services/profile.service.ts
View File

@@ -64,17 +64,16 @@ interface GetProfileListOptions {
}
export async function getProfiles(ids: string[]) {
if (ids.length === 0) {
const filteredIds = ids.filter((id) => id !== '');
if (filteredIds.length === 0) {
return [];
}
const data = await chQuery<IClickhouseProfile>(
`SELECT *
FROM profiles FINAL
WHERE id IN (${ids
.map((id) => escape(id))
.filter(Boolean)
.join(',')})
WHERE id IN (${filteredIds.map((id) => escape(id)).join(',')})
`
);