a looooot

This commit is contained in:
Carl-Gerhard Lindesvärd
2024-02-22 21:50:30 +01:00
parent 1d800835b8
commit 9c92803c4c
61 changed files with 2689 additions and 681 deletions

View File

@@ -1,4 +1,4 @@
import { anyPass, isEmpty, isNil, reject } from 'ramda';
import { anyPass, assocPath, isEmpty, isNil, reject } from 'ramda';
export function toDots(
obj: Record<string, unknown>,
@@ -19,6 +19,16 @@ export function toDots(
}, {});
}
export function toObject(
obj: Record<string, string | undefined>
): Record<string, unknown> {
let result: Record<string, unknown> = {};
Object.entries(obj).forEach(([key, value]) => {
result = assocPath(key.split('.'), value, result);
});
return result;
}
export const strip = reject(anyPass([isEmpty, isNil]));
export function getSafeJson<T>(str: string): T | null {

View File

@@ -1,17 +1,17 @@
import { createHash } from './crypto';
interface GenerateProfileIdOptions {
interface GenerateDeviceIdOptions {
salt: string;
ua: string;
ip: string;
origin: string;
}
export function generateProfileId({
export function generateDeviceId({
salt,
ua,
ip,
origin,
}: GenerateProfileIdOptions) {
}: GenerateDeviceIdOptions) {
return createHash(`${ua}:${ip}:${origin}:${salt}`, 16);
}

View File

@@ -1,6 +1,7 @@
CREATE TABLE openpanel.events (
`id` UUID DEFAULT generateUUIDv4(),
`name` String,
`device_id` String,
`profile_id` String,
`project_id` String,
`path` String,
@@ -28,7 +29,17 @@ CREATE TABLE openpanel.events (
ORDER BY
(project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
CREATE TABLE test.profiles (
CREATE TABLE openpanel.events_bots (
`project_id` String,
`name` String,
`type` String,
`path` String,
`created_at` DateTime64(3),
) ENGINE MergeTree
ORDER BY
(project_id, created_at) SETTINGS index_granularity = 8192;
CREATE TABLE profiles (
`id` String,
`external_id` String,
`first_name` String,
@@ -38,16 +49,24 @@ CREATE TABLE test.profiles (
`properties` Map(String, String),
`project_id` String,
`created_at` DateTime
) ENGINE = ReplacingMergeTree
) ENGINE = ReplacingMergeTree(created_at)
ORDER BY
(id) SETTINGS index_granularity = 8192;
ALTER TABLE
events
ADD
COLUMN continent String
COLUMN device_id String
AFTER
region;
name;
ALTER TABLE
events DROP COLUMN id;
events DROP COLUMN id;
CREATE TABLE ba (
`id` UUID DEFAULT generateUUIDv4(),
`a` String,
`b` String
) ENGINE MergeTree
ORDER BY
(a, b) SETTINGS index_granularity = 8192;

View File

@@ -15,11 +15,13 @@ import type { EventMeta, Prisma } from '../prisma-client';
import { db } from '../prisma-client';
import { createSqlBuilder } from '../sql-builder';
import { getEventFiltersWhereClause } from './chart.service';
import { getProfileById, getProfiles, upsertProfile } from './profile.service';
import type { IServiceProfile } from './profile.service';
export interface IClickhouseEvent {
id: string;
name: string;
device_id: string;
profile_id: string;
project_id: string;
path: string;
@@ -51,6 +53,7 @@ export function transformEvent(
return {
id: event.id,
name: event.name,
deviceId: event.device_id,
profileId: event.profile_id,
projectId: event.project_id,
properties: event.properties,
@@ -78,6 +81,7 @@ export function transformEvent(
export interface IServiceCreateEventPayload {
id: string;
name: string;
deviceId: string;
profileId: string;
projectId: string;
properties: Record<string, unknown> & {
@@ -121,15 +125,8 @@ export async function getEvents(
): Promise<IServiceCreateEventPayload[]> {
const events = await chQuery<IClickhouseEvent>(sql);
if (options.profile) {
const profileIds = events.map((e) => e.profile_id);
const profiles = await db.profile.findMany({
where: {
id: {
in: profileIds,
},
},
select: options.profile === true ? undefined : options.profile,
});
const ids = events.map((e) => e.profile_id);
const profiles = await getProfiles({ ids });
for (const event of events) {
event.profile = profiles.find((p) => p.id === event.profile_id);
@@ -157,41 +154,38 @@ export async function getEvents(
export async function createEvent(
payload: Omit<IServiceCreateEventPayload, 'id'>
) {
console.log(`create event ${payload.name} for ${payload.profileId}`);
if (!payload.profileId) {
payload.profileId = payload.deviceId;
}
console.log(
`create event ${payload.name} for deviceId: ${payload.deviceId} profileId ${payload.profileId}`
);
if (payload.name === 'session_start') {
const profile = await db.profile.findUnique({
where: {
id: payload.profileId,
const exists = await getProfileById(payload.profileId);
if (!exists) {
const { firstName, lastName } = randomSplitName();
await upsertProfile({
id: payload.profileId,
projectId: payload.projectId,
firstName,
lastName,
properties: {
path: payload.path,
country: payload.country,
city: payload.city,
region: payload.region,
os: payload.os,
os_version: payload.osVersion,
browser: payload.browser,
browser_version: payload.browserVersion,
device: payload.device,
brand: payload.brand,
model: payload.model,
referrer: payload.referrer,
referrer_name: payload.referrerName,
referrer_type: payload.referrerType,
},
});
if (!profile) {
const { firstName, lastName } = randomSplitName();
await db.profile.create({
data: {
id: payload.profileId,
project_id: payload.projectId,
first_name: firstName,
last_name: lastName,
properties: {
country: payload.country ?? '',
city: payload.city ?? '',
region: payload.region ?? '',
os: payload.os ?? '',
os_version: payload.osVersion ?? '',
browser: payload.browser ?? '',
browser_version: payload.browserVersion ?? '',
device: payload.device ?? '',
brand: payload.brand ?? '',
model: payload.model ?? '',
referrer: payload.referrer ?? '',
referrer_name: payload.referrerName ?? '',
referrer_type: payload.referrerType ?? '',
},
},
});
}
}
if (payload.properties.hash === '') {
@@ -201,6 +195,7 @@ export async function createEvent(
const event: IClickhouseEvent = {
id: uuid(),
name: payload.name,
device_id: payload.deviceId,
profile_id: payload.profileId,
project_id: payload.projectId,
properties: toDots(omit(['_path'], payload.properties)),
@@ -245,7 +240,7 @@ export async function createEvent(
};
}
interface GetEventListOptions {
export interface GetEventListOptions {
projectId: string;
profileId?: string;
take: number;
@@ -321,3 +316,38 @@ export async function getEventsCount({
return res[0]?.count ?? 0;
}
interface CreateBotEventPayload {
name: string;
type: string;
projectId: string;
createdAt: Date;
}
export function createBotEvent({
name,
type,
projectId,
createdAt,
}: CreateBotEventPayload) {
return ch.insert({
table: 'events_bots',
values: [
{
name,
type,
project_id: projectId,
created_at: formatClickhouseDate(createdAt),
},
],
});
}
export function getConversionEventNames(projectId: string) {
return db.eventMeta.findMany({
where: {
project_id: projectId,
conversion: true,
},
});
}

View File

@@ -1,16 +1,103 @@
import { db } from '../prisma-client';
import { toDots, toObject } from '@mixan/common';
import type { IChartEventFilter } from '@mixan/validation';
export type IServiceProfile = Awaited<ReturnType<typeof getProfileById>>;
import { ch, chQuery } from '../clickhouse-client';
import { createSqlBuilder } from '../sql-builder';
import { getEventFiltersWhereClause } from './chart.service';
export function getProfileById(id: string) {
return db.profile.findUniqueOrThrow({
where: {
id,
},
});
export async function getProfileById(id: string) {
const [profile] = await chQuery<IClickhouseProfile>(
`SELECT * FROM profiles WHERE id = '${id}' ORDER BY created_at DESC LIMIT 1`
);
if (!profile) {
return null;
}
return transformProfile(profile);
}
export function getProfilesByExternalId(
interface GetProfileListOptions {
projectId: string;
take: number;
cursor?: number;
filters?: IChartEventFilter[];
}
function getProfileSelectFields() {
return [
'id',
'argMax(first_name, created_at) as first_name',
'argMax(last_name, created_at) as last_name',
'argMax(email, created_at) as email',
'argMax(avatar, created_at) as avatar',
'argMax(properties, created_at) as properties',
'argMax(project_id, created_at) as project_id',
'max(created_at) as max_created_at',
].join(', ');
}
interface GetProfilesOptions {
ids: string[];
}
export async function getProfiles({ ids }: GetProfilesOptions) {
if (ids.length === 0) {
return [];
}
const data = await chQuery<IClickhouseProfile>(
`SELECT
${getProfileSelectFields()}
FROM profiles
WHERE id IN (${ids.map((id) => `'${id}'`).join(',')})
GROUP BY id
`
);
return data.map(transformProfile);
}
function getProfileInnerSelect(projectId: string) {
return `(SELECT
${getProfileSelectFields()}
FROM profiles
GROUP BY id
HAVING project_id = '${projectId}')`;
}
export async function getProfileList({
take,
cursor,
projectId,
filters,
}: GetProfileListOptions) {
const { sb, getSql } = createSqlBuilder();
sb.from = getProfileInnerSelect(projectId);
if (filters) {
getEventFiltersWhereClause(sb, filters);
}
sb.limit = take;
sb.offset = (cursor ?? 0) * take;
sb.orderBy.created_at = 'max_created_at DESC';
const data = await chQuery<IClickhouseProfile>(getSql());
return data.map(transformProfile);
}
export async function getProfileListCount({
projectId,
filters,
}: Omit<GetProfileListOptions, 'cursor' | 'take'>) {
const { sb, getSql } = createSqlBuilder();
sb.select.count = 'count(id) as count';
sb.from = getProfileInnerSelect(projectId);
if (filters) {
getEventFiltersWhereClause(sb, filters);
}
const [data] = await chQuery<{ count: number }>(getSql());
return data?.count ?? 0;
}
export async function getProfilesByExternalId(
externalId: string | null,
projectId: string
) {
@@ -18,18 +105,91 @@ export function getProfilesByExternalId(
return [];
}
return db.profile.findMany({
where: {
external_id: externalId,
project_id: projectId,
},
});
const data = await chQuery<IClickhouseProfile>(
`SELECT
${getProfileSelectFields()}
FROM profiles
GROUP BY id
HAVING project_id = '${projectId}' AND external_id = '${externalId}'
`
);
return data.map(transformProfile);
}
export function getProfile(id: string) {
return db.profile.findUniqueOrThrow({
where: {
id,
export type IServiceProfile = Omit<
IClickhouseProfile,
'max_created_at' | 'properties'
> & {
createdAt: Date;
properties: Record<string, unknown>;
};
export interface IClickhouseProfile {
id: string;
first_name: string;
last_name: string;
email: string;
avatar: string;
properties: Record<string, string | undefined>;
project_id: string;
max_created_at: string;
}
export interface IServiceUpsertProfile {
projectId: string;
id: string;
firstName?: string;
lastName?: string;
email?: string;
avatar?: string;
properties?: Record<string, unknown>;
}
function transformProfile({
max_created_at,
...profile
}: IClickhouseProfile): IServiceProfile {
return {
...profile,
properties: toObject(profile.properties),
createdAt: new Date(max_created_at),
};
}
export async function upsertProfile({
id,
firstName,
lastName,
email,
avatar,
properties,
projectId,
}: IServiceUpsertProfile) {
const [profile] = await chQuery<IClickhouseProfile>(
`SELECT * FROM profiles WHERE id = '${id}' AND project_id = '${projectId}' ORDER BY created_at DESC LIMIT 1`
);
await ch.insert({
table: 'profiles',
format: 'JSONEachRow',
clickhouse_settings: {
date_time_input_format: 'best_effort',
},
values: [
{
id,
first_name: firstName ?? profile?.first_name ?? '',
last_name: lastName ?? profile?.last_name ?? '',
email: email ?? profile?.email ?? '',
avatar: avatar ?? profile?.avatar ?? '',
properties: toDots({
...(profile?.properties ?? {}),
...(properties ?? {}),
}),
project_id: projectId ?? profile?.project_id ?? '',
created_at: new Date(),
},
],
});
}

View File

@@ -10,7 +10,7 @@ export interface EventsQueuePayloadCreateEvent {
}
export interface EventsQueuePayloadCreateSessionEnd {
type: 'createSessionEnd';
payload: Pick<IServiceCreateEventPayload, 'profileId'>;
payload: Pick<IServiceCreateEventPayload, 'deviceId'>;
}
export type EventsQueuePayload =
| EventsQueuePayloadCreateEvent

View File

@@ -5,9 +5,7 @@ import Constants from 'expo-constants';
import type { MixanOptions } from '@mixan/sdk';
import { Mixan } from '@mixan/sdk';
type MixanNativeOptions = MixanOptions & {
ipUrl?: string;
};
type MixanNativeOptions = MixanOptions
export class MixanNative extends Mixan<MixanNativeOptions> {
constructor(options: MixanNativeOptions) {

110
packages/sdk-next/index.tsx Normal file
View File

@@ -0,0 +1,110 @@
import Script from 'next/script';
import type { MixanEventOptions } from '@mixan/sdk';
import type { MixanWebOptions } from '@mixan/sdk-web';
import type { UpdateProfilePayload } from '@mixan/types';
const CDN_URL = 'http://localhost:3002/op.js';
type OpenpanelMethods =
| 'ctor'
| 'event'
| 'setProfile'
| 'setProfileId'
| 'increment'
| 'decrement'
| 'clear';
declare global {
interface Window {
op: {
q?: [string, ...any[]];
(method: OpenpanelMethods, ...args: any[]): void;
};
}
}
type OpenpanelProviderProps = MixanWebOptions & {
profileId?: string;
cdnUrl?: string;
};
export function OpenpanelProvider({
profileId,
cdnUrl,
...options
}: OpenpanelProviderProps) {
const events: { name: OpenpanelMethods; value: unknown }[] = [
{ name: 'ctor', value: options },
];
if (profileId) {
events.push({ name: 'setProfileId', value: profileId });
}
return (
<>
<Script src={cdnUrl ?? CDN_URL} async defer />
<Script
dangerouslySetInnerHTML={{
__html: `window.op = window.op || function(...args) {(window.op.q = window.op.q || []).push(args)};
${events
.map((event) => {
return `window.op('${event.name}', ${JSON.stringify(event.value)});`;
})
.join('\n')}`,
}}
/>
</>
);
}
interface SetProfileIdProps {
value?: string;
}
export function SetProfileId({ value }: SetProfileIdProps) {
return (
<>
<Script
dangerouslySetInnerHTML={{
__html: `window.op('setProfileId', '${value}');`,
}}
/>
</>
);
}
export function trackEvent(name: string, data?: Record<string, unknown>) {
window.op('event', name, data);
}
export function trackScreenView(data?: Record<string, unknown>) {
trackEvent('screen_view', data);
}
export function setProfile(data?: UpdateProfilePayload) {
window.op('setProfile', data);
}
export function setProfileId(profileId: string) {
window.op('setProfileId', profileId);
}
export function increment(
property: string,
value: number,
options?: MixanEventOptions
) {
window.op('increment', property, value, options);
}
export function decrement(
property: string,
value: number,
options?: MixanEventOptions
) {
window.op('decrement', property, value, options);
}
export function clear() {
window.op('clear');
}

View File

@@ -0,0 +1,36 @@
{
"name": "@mixan/next",
"version": "0.0.1",
"module": "index.ts",
"scripts": {
"build": "rm -rf dist && tsup",
"build-for-openpanel": "pnpm build && cp dist/cdn.global.js ../../apps/public/public/op.js && cp dist/cdn.global.js ../../apps/test/public/op.js",
"lint": "eslint .",
"format": "prettier --check \"**/*.{mjs,ts,md,json}\"",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@mixan/sdk": "workspace:*",
"@mixan/sdk-web": "workspace:*",
"@mixan/types": "workspace:*"
},
"peerDependencies": {
"next": "^13.0.0"
},
"devDependencies": {
"@mixan/eslint-config": "workspace:*",
"@mixan/prettier-config": "workspace:*",
"@mixan/tsconfig": "workspace:*",
"eslint": "^8.48.0",
"prettier": "^3.0.3",
"tsup": "^7.2.0",
"typescript": "^5.2.2"
},
"eslintConfig": {
"root": true,
"extends": [
"@mixan/eslint-config/base"
]
},
"prettier": "@mixan/prettier-config"
}

View File

@@ -0,0 +1,6 @@
{
"extends": "@mixan/tsconfig/base.json",
"compilerOptions": {
"outDir": "dist"
}
}

View File

@@ -0,0 +1,9 @@
import { defineConfig } from 'tsup';
import config from '@mixan/tsconfig/tsup.config.json' assert { type: 'json' };
export default defineConfig({
...(config as any),
entry: ['index.ts'],
format: ['cjs', 'esm'],
});

View File

@@ -1,13 +1,31 @@
// @ts-nocheck
import { MixanWeb as Openpanel } from './index';
const el = document.currentScript;
if (el) {
window.openpanel = new Openpanel({
url: el?.getAttribute('data-url'),
clientId: el?.getAttribute('data-client-id'),
trackOutgoingLinks: !!el?.getAttribute('data-track-outgoing-links'),
trackScreenViews: !!el?.getAttribute('data-track-screen-views'),
});
declare global {
interface Window {
op: {
q?: [string, ...any[]];
(method: string, ...args: any[]): void;
};
}
}
((window) => {
if (window.op && 'q' in window.op) {
const queue = window.op.q || [];
const op = new Openpanel(queue.shift()[1]);
queue.forEach((item) => {
if (item[0] in op) {
// @ts-expect-error
op[item[0]](...item.slice(1));
}
});
window.op = (t, ...args) => {
// @ts-expect-error
const fn = op[t].bind(op);
if (typeof fn === 'function') {
fn(...args);
}
};
}
})(window);

View File

@@ -1,12 +1,19 @@
import type { MixanOptions } from '@mixan/sdk';
import { Mixan } from '@mixan/sdk';
type MixanWebOptions = MixanOptions & {
export type MixanWebOptions = MixanOptions & {
trackOutgoingLinks?: boolean;
trackScreenViews?: boolean;
trackAttributes?: boolean;
hash?: boolean;
};
function toCamelCase(str: string) {
return str.replace(/([-_][a-z])/gi, ($1) =>
$1.toUpperCase().replace('-', '').replace('_', '')
);
}
export class MixanWeb extends Mixan<MixanWebOptions> {
private lastPath = '';
@@ -25,6 +32,10 @@ export class MixanWeb extends Mixan<MixanWebOptions> {
if (this.options.trackScreenViews) {
this.trackScreenViews();
}
if (this.options.trackAttributes) {
this.trackAttributes();
}
}
}
@@ -87,7 +98,40 @@ export class MixanWeb extends Mixan<MixanWebOptions> {
window.addEventListener('locationchange', () => this.screenView());
}
this.screenView();
// give time for setProfile to be called
setTimeout(() => {
this.screenView();
}, 50);
}
public trackAttributes() {
if (this.isServer()) {
return;
}
document.addEventListener('click', (event) => {
const target = event.target as HTMLElement;
const btn = target.closest('button');
const achor = target.closest('button');
const element = btn?.getAttribute('data-event')
? btn
: achor?.getAttribute('data-event')
? achor
: null;
if (element) {
const properties: Record<string, unknown> = {};
for (const attr of element.attributes) {
if (attr.name.startsWith('data-') && attr.name !== 'data-event') {
properties[toCamelCase(attr.name.replace(/^data-/, ''))] =
attr.value;
}
}
const name = element.getAttribute('data-event');
if (name) {
super.event(name, properties);
}
}
});
}
public screenView(properties?: Record<string, unknown>): void {

View File

@@ -10,16 +10,21 @@ export interface MixanOptions {
clientId: string;
clientSecret?: string;
verbose?: boolean;
setProfileId?: (profileId: string) => void;
getProfileId?: () => string | null | undefined;
removeProfileId?: () => void;
setDeviceId?: (deviceId: string) => void;
getDeviceId?: () => string | null | undefined;
removeDeviceId?: () => void;
}
export interface MixanState {
deviceId?: string;
profileId?: string;
properties: Record<string, unknown>;
}
export interface MixanEventOptions {
profileId?: string;
}
function awaitProperties(
properties: Record<string, string | Promise<string | null>>
): Promise<Record<string, string>> {
@@ -55,6 +60,10 @@ function createApi(_url: string) {
...(options ?? {}),
})
.then(async (res) => {
if (res.status === 401) {
return null;
}
if (res.status !== 200 && res.status !== 202) {
return retry(attempt, resolve);
}
@@ -116,9 +125,13 @@ export class Mixan<Options extends MixanOptions = MixanOptions> {
this.state.properties = properties ?? {};
}
public setUser(payload: Omit<UpdateProfilePayload, 'profileId'>) {
public setProfileId(profileId: string) {
this.state.profileId = profileId;
}
public setProfile(payload: UpdateProfilePayload) {
this.setProfileId(payload.profileId);
this.api.fetch<UpdateProfilePayload, string>('/profile', {
profileId: this.getProfileId(),
...payload,
properties: {
...this.state.properties,
@@ -127,23 +140,44 @@ export class Mixan<Options extends MixanOptions = MixanOptions> {
});
}
public increment(property: string, value: number) {
public increment(
property: string,
value: number,
options?: MixanEventOptions
) {
const profileId = options?.profileId ?? this.state.profileId;
if (!profileId) {
return console.log('No profile id');
}
this.api.fetch<IncrementProfilePayload, string>('/profile/increment', {
profileId,
property,
value,
profileId: this.getProfileId(),
});
}
public decrement(property: string, value: number) {
public decrement(
property: string,
value: number,
options?: MixanEventOptions
) {
const profileId = options?.profileId ?? this.state.profileId;
if (!profileId) {
return console.log('No profile id');
}
this.api.fetch<DecrementProfilePayload, string>('/profile/decrement', {
profileId,
property,
value,
profileId: this.getProfileId(),
});
}
public event(name: string, properties?: Record<string, unknown>) {
public event(
name: string,
properties?: Record<string, unknown> & MixanEventOptions
) {
const profileId = properties?.profileId ?? this.state.profileId;
delete properties?.profileId;
this.api
.fetch<PostEventPayload, string>('/event', {
name,
@@ -152,11 +186,12 @@ export class Mixan<Options extends MixanOptions = MixanOptions> {
...(properties ?? {}),
},
timestamp: this.timestamp(),
profileId: this.getProfileId(),
deviceId: this.getDeviceId(),
profileId,
})
.then((profileId) => {
if (this.options.setProfileId && profileId) {
this.options.setProfileId(profileId);
.then((deviceId) => {
if (this.options.setDeviceId && deviceId) {
this.options.setDeviceId(deviceId);
}
});
}
@@ -169,9 +204,10 @@ export class Mixan<Options extends MixanOptions = MixanOptions> {
}
public clear() {
this.state.profileId = undefined;
if (this.options.removeProfileId) {
this.options.removeProfileId();
this.state.properties = {};
this.state.deviceId = undefined;
if (this.options.removeDeviceId) {
this.options.removeDeviceId();
}
}
@@ -181,11 +217,11 @@ export class Mixan<Options extends MixanOptions = MixanOptions> {
return new Date().toISOString();
}
private getProfileId() {
if (this.state.profileId) {
return this.state.profileId;
} else if (this.options.getProfileId) {
this.state.profileId = this.options.getProfileId() || undefined;
private getDeviceId() {
if (this.state.deviceId) {
return this.state.deviceId;
} else if (this.options.getDeviceId) {
this.state.deviceId = this.options.getDeviceId() || undefined;
}
}
}

View File

@@ -137,6 +137,7 @@ export interface MixanResponse<T> {
export interface PostEventPayload {
name: string;
timestamp: string;
deviceId?: string;
profileId?: string;
properties?: Record<string, unknown> & {
title?: string | undefined;
@@ -146,17 +147,16 @@ export interface PostEventPayload {
}
export interface UpdateProfilePayload {
profileId?: string;
id?: string;
first_name?: string;
last_name?: string;
profileId: string;
firstName?: string;
lastName?: string;
email?: string;
avatar?: string;
properties?: MixanJson;
}
export interface IncrementProfilePayload {
profileId?: string;
profileId: string;
property: string;
value: number;
}