sdk changes

packages/common/index.ts

@@ -2,3 +2,4 @@ export * from './src/crypto';
 export * from './src/profileId';
 export * from './src/date';
 export * from './src/object';
+export * from './src/names';

packages/common/package.json

@@ -8,7 +8,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "ramda": "^0.29.1"
+    "ramda": "^0.29.1",
+    "unique-names-generator": "^4.7.1"
   },
   "devDependencies": {
     "@mixan/eslint-config": "workspace:*",

packages/common/src/crypto.ts

@@ -1,4 +1,9 @@
-import { randomBytes, scrypt, timingSafeEqual } from 'crypto';
+import {
+  createHash as cryptoCreateHash,
+  randomBytes,
+  scrypt,
+  timingSafeEqual,
+} from 'crypto';
 
 export function generateSalt() {
   return randomBytes(16).toString('hex');
@@ -11,12 +16,11 @@ export function generateSalt() {
  */
 export async function hashPassword(
   password: string,
-  _salt?: string,
   keyLength = 32
 ): Promise<string> {
   return new Promise((resolve, reject) => {
     // generate random 16 bytes long salt - recommended by NodeJS Docs
-    const salt = _salt || generateSalt();
+    const salt = generateSalt();
     scrypt(password, salt, keyLength, (err, derivedKey) => {
       if (err) reject(err);
       // derivedKey is of type Buffer
@@ -49,3 +53,9 @@ export async function verifyPassword(
     });
   });
 }
+
+export function createHash(data: string, len: number) {
+  return cryptoCreateHash('shake256', { outputLength: len })
+    .update(data)
+    .digest('hex');
+}

packages/common/src/names.ts

@@ -0,0 +1,18 @@
+import { animals, names, uniqueNamesGenerator } from 'unique-names-generator';
+
+export function randomName() {
+  return uniqueNamesGenerator({
+    dictionaries: [names, animals],
+    length: 2,
+    style: 'capital',
+    separator: ' ',
+  });
+}
+
+export function randomSplitName() {
+  const [firstName, lastName] = randomName().split(' ');
+  return {
+    firstName,
+    lastName,
+  };
+}

packages/common/src/profileId.ts

@@ -1,4 +1,4 @@
-import { hashPassword } from './crypto';
+import { createHash } from './crypto';
 
 interface GenerateProfileIdOptions {
   salt: string;
@@ -7,11 +7,11 @@ interface GenerateProfileIdOptions {
   origin: string;
 }
 
-export async function generateProfileId({
+export function generateProfileId({
   salt,
   ua,
   ip,
   origin,
 }: GenerateProfileIdOptions) {
-  return await hashPassword(`${ua}:${ip}:${origin}`, salt, 8);
+  return createHash(`${ua}:${ip}:${origin}:${salt}`, 16);
 }

packages/db/clickhouse_tables.sql

@@ -2,15 +2,10 @@ CREATE TABLE openpanel.events (
   `name` String,
   `profile_id` String,
   `project_id` String,
-  -- the route
   `path` String,
-  `utm_source` String,
-  `utm_medium` String,
-  `utm_campaign` String,
-  `utm_term` String,
-  `utm_content` String,
   `referrer` String,
   `referrer_name` String,
+  `referrer_type` String,
   `duration` UInt64,
   `properties` Map(String, String),
   `created_at` DateTime64(3),

packages/db/scripts/insert.ts

@@ -97,24 +97,23 @@ async function main() {
         const ua = event.properties.ua as string;
         const uaInfo = parseUserAgent(ua);
         const salts = await getSalts();
+        const currentProfileId = generateProfileId({
+          salt: salts.current,
+          origin,
+          ip,
+          ua,
+        });
+        const previousProfileId = generateProfileId({
+          salt: salts.previous,
+          origin,
+          ip,
+          ua,
+        });
 
-        const [currentProfileId, previousProfileId, geo, eventsJobs] =
-          await Promise.all([
-            generateProfileId({
-              salt: salts.current,
-              origin,
-              ip,
-              ua,
-            }),
-            generateProfileId({
-              salt: salts.previous,
-              origin,
-              ip,
-              ua,
-            }),
-            parseIp(ip),
-            eventsQueue.getJobs(['delayed']),
-          ]);
+        const [geo, eventsJobs] = Promise.all([
+          parseIp(ip),
+          eventsQueue.getJobs(['delayed']),
+        ]);
         const payload: IServiceCreateEventPayload = {
           name: body.name,
           profileId,

packages/db/src/prisma-types.ts

@@ -1,5 +1,7 @@
 /* eslint-disable */
 
+import { Profile } from '@prisma/client';
+
 export type IDBEvent = {
   id: string;
   name: string;
@@ -9,14 +11,6 @@ export type IDBEvent = {
   created_at: string;
 };
 
-export type IDBProfile = {
-  id: string;
-  external_id?: string;
-  first_name?: string;
-  last_name?: string;
-  email?: string;
-  avatar?: string;
-  properties: Record<string, string>;
-  project_id: String;
-  created_at: string;
+export type IDBProfile = Omit<Profile, 'properties'> & {
+  properties: Record<string, unknown>;
 };

packages/db/src/services/event.service.ts

@@ -1,9 +1,10 @@
 import { omit } from 'ramda';
 
-import { toDots } from '@mixan/common';
-import { redisPub } from '@mixan/redis';
+import { randomSplitName, toDots } from '@mixan/common';
+import { redis, redisPub } from '@mixan/redis';
 
 import { ch, chQuery, formatClickhouseDate } from '../clickhouse-client';
+import { db } from '../prisma-client';
 
 export interface IClickhouseEvent {
   name: string;
@@ -12,6 +13,7 @@ export interface IClickhouseEvent {
   path: string;
   referrer: string;
   referrer_name: string;
+  referrer_type: string;
   duration: number;
   properties: Record<string, string | number | boolean>;
   created_at: string;
@@ -50,6 +52,7 @@ export function transformEvent(
     path: event.path,
     referrer: event.referrer,
     referrerName: event.referrer_name,
+    referrerType: event.referrer_type,
   };
 }
 
@@ -77,6 +80,7 @@ export interface IServiceCreateEventPayload {
   path: string;
   referrer: string | undefined;
   referrerName: string | undefined;
+  referrerType: string | undefined;
 }
 
 export function getEvents(sql: string) {
@@ -88,6 +92,41 @@ export function getEvents(sql: string) {
 export async function createEvent(payload: IServiceCreateEventPayload) {
   console.log(`create event ${payload.name} for ${payload.profileId}`);
 
+  if (payload.name === 'session_start') {
+    const profile = await db.profile.findUnique({
+      where: {
+        id: payload.profileId,
+      },
+    });
+
+    if (!profile) {
+      const { firstName, lastName } = randomSplitName();
+      await db.profile.create({
+        data: {
+          id: payload.profileId,
+          project_id: payload.projectId,
+          first_name: firstName,
+          last_name: lastName,
+          properties: {
+            country: payload.country ?? '',
+            city: payload.city ?? '',
+            region: payload.region ?? '',
+            os: payload.os ?? '',
+            os_version: payload.osVersion ?? '',
+            browser: payload.browser ?? '',
+            browser_version: payload.browserVersion ?? '',
+            device: payload.device ?? '',
+            brand: payload.brand ?? '',
+            model: payload.model ?? '',
+            referrer: payload.referrer ?? '',
+            referrer_name: payload.referrerName ?? '',
+            referrer_type: payload.referrerType ?? '',
+          },
+        },
+      });
+    }
+  }
+
   if (payload.properties.hash === '') {
     delete payload.properties.hash;
   }
@@ -112,6 +151,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
     duration: payload.duration,
     referrer: payload.referrer ?? '',
     referrer_name: payload.referrerName ?? '',
+    referrer_type: payload.referrerType ?? '',
   };
 
   const res = await ch.insert({
@@ -121,6 +161,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
   });
 
   redisPub.publish('event', JSON.stringify(transformEvent(event)));
+  redis.set(`live:event:${event.project_id}:${event.profile_id}`, '', 'EX', 10);
 
   return {
     ...res,

packages/sdk-native/index.ts

@@ -1,21 +1,46 @@
-import type { NewMixanOptions } from '@mixan/sdk';
+import { AppState, Platform } from 'react-native';
+import * as Application from 'expo-application';
+import Constants from 'expo-constants';
+import * as Network from 'expo-network';
+
+import type { MixanOptions } from '@mixan/sdk';
 import { Mixan } from '@mixan/sdk';
 
-export class MixanNative extends Mixan {
-  constructor(options: NewMixanOptions) {
+type MixanNativeOptions = MixanOptions & {
+  ipUrl?: string;
+};
+
+export class MixanNative extends Mixan<MixanNativeOptions> {
+  constructor(options: MixanNativeOptions) {
     super(options);
+
+    this.api.headers['X-Forwarded-For'] = Network.getIpAddressAsync();
+    this.api.headers['User-Agent'] = Constants.getWebViewUserAgentAsync();
+
+    AppState.addEventListener('change', (state) => {
+      if (state === 'active') {
+        this.setProperties();
+      }
+    });
+
+    this.setProperties();
   }
 
-  init(properties?: Record<string, unknown>) {
-    super.init({
-      ...(properties ?? {}),
+  private async setProperties() {
+    this.setGlobalProperties({
+      version: Application.nativeApplicationVersion,
+      buildNumber: Application.nativeBuildVersion,
+      referrer:
+        Platform.OS === 'android'
+          ? await Application.getInstallReferrerAsync()
+          : undefined,
     });
   }
 
-  screenView(route: string, properties?: Record<string, unknown>): void {
+  public screenView(route: string, properties?: Record<string, unknown>): void {
     super.event('screen_view', {
       ...properties,
-      route: route,
+      path: route,
     });
   }
 }

packages/sdk-native/package.json

@@ -21,6 +21,12 @@
     "tsup": "^7.2.0",
     "typescript": "^5.2.2"
   },
+  "peerDependencies": {
+    "react-native": "^0.72.5",
+    "expo-application": "~5.3.0",
+    "expo-constants": "~14.4.2",
+    "expo-network": "~5.8.0"
+  },
   "eslintConfig": {
     "root": true,
     "extends": [

packages/sdk-native/tsup.config.ts

@@ -1,7 +1,8 @@
 import { defineConfig } from 'tsup';
 
-import config from '@mixan/tsconfig/tsup.config.json' assert {
-  type: 'json'
-}
+import config from '@mixan/tsconfig/tsup.config.json' assert { type: 'json' };
 
-export default defineConfig(config as any);
+export default defineConfig({
+  ...(config as any),
+  minify: false,
+});

packages/sdk-web/cdn.ts

@@ -7,7 +7,6 @@ if (el) {
   window.openpanel = new Openpanel({
     url: el?.getAttribute('data-url'),
     clientId: el?.getAttribute('data-client-id'),
-    clientSecret: el?.getAttribute('data-client-secret'),
     trackOutgoingLinks: !!el?.getAttribute('data-track-outgoing-links'),
     trackScreenViews: !!el?.getAttribute('data-track-screen-views'),
   });

packages/sdk-web/index.ts

@@ -8,6 +8,8 @@ type MixanWebOptions = MixanOptions & {
 };
 
 export class MixanWeb extends Mixan<MixanWebOptions> {
+  private lastPath = '';
+
   constructor(options: MixanWebOptions) {
     super(options);
 
@@ -24,14 +26,6 @@ export class MixanWeb extends Mixan<MixanWebOptions> {
     return typeof document === 'undefined';
   }
 
-  private getTimezone() {
-    try {
-      return Intl.DateTimeFormat().resolvedOptions().timeZone;
-    } catch (e) {
-      return undefined;
-    }
-  }
-
   public trackOutgoingLinks() {
     if (this.isServer()) {
       return;
@@ -90,9 +84,16 @@ export class MixanWeb extends Mixan<MixanWebOptions> {
       return;
     }
 
+    const path = window.location.href;
+
+    if (this.lastPath === path) {
+      return;
+    }
+
+    this.lastPath = path;
     super.event('screen_view', {
       ...(properties ?? {}),
-      path: window.location.href,
+      path,
       title: document.title,
       referrer: document.referrer,
     });

packages/sdk-web/package.json

@@ -4,7 +4,7 @@
   "module": "index.ts",
   "scripts": {
     "build": "rm -rf dist && tsup",
-    "build-for-openpanel": "pnpm build && cp dist/cdn.global.js ../../apps/public/public/op.js",
+    "build-for-openpanel": "pnpm build && cp dist/cdn.global.js ../../apps/public/public/op.js && cp dist/cdn.global.js ../../apps/test/public/op.js",
     "lint": "eslint .",
     "format": "prettier --check \"**/*.{mjs,ts,md,json}\"",
     "typecheck": "tsc --noEmit"

packages/sdk-web/tsup.config.ts

@@ -1,7 +1,9 @@
 import { defineConfig } from 'tsup';
 
-import config from '@mixan/tsconfig/tsup.config.json' assert {
-  type: 'json'
-}
+import config from '@mixan/tsconfig/tsup.config.json' assert { type: 'json' };
 
-export default defineConfig(config as any);
+export default defineConfig({
+  ...(config as any),
+  entry: ['index.ts', 'cdn.ts'],
+  format: ['cjs', 'esm', 'iife'],
+});

packages/sdk/index.ts

@@ -1,4 +1,9 @@
-import type { PostEventPayload } from '@mixan/types';
+import type {
+  DecrementProfilePayload,
+  IncrementProfilePayload,
+  PostEventPayload,
+  UpdateProfilePayload,
+} from '@mixan/types';
 
 export interface MixanOptions {
   url: string;
@@ -15,80 +20,94 @@ export interface MixanState {
   properties: Record<string, unknown>;
 }
 
-function createApi(_url: string, clientId: string, clientSecret?: string) {
-  return function post<ReqBody, ResBody>(
-    path: string,
-    data: ReqBody,
-    options?: RequestInit
-  ): Promise<ResBody | null> {
-    const url = `${_url}${path}`;
-    let timer: ReturnType<typeof setTimeout>;
-    const headers: Record<string, string> = {
-      'mixan-client-id': clientId,
-      'Content-Type': 'application/json',
-    };
-    if (clientSecret) {
-      headers['mixan-client-secret'] = clientSecret;
-    }
-    return new Promise((resolve) => {
-      const wrappedFetch = (attempt: number) => {
-        clearTimeout(timer);
-        fetch(url, {
-          headers,
-          method: 'POST',
-          body: JSON.stringify(data ?? {}),
-          keepalive: true,
-          ...(options ?? {}),
-        })
-          .then(async (res) => {
-            if (res.status !== 200 && res.status !== 202) {
-              return retry(attempt, resolve);
-            }
+function awaitProperties(
+  properties: Record<string, string | Promise<string | null>>
+): Promise<Record<string, string>> {
+  return Promise.all(
+    Object.entries(properties).map(async ([key, value]) => {
+      return [key, (await value) ?? ''];
+    })
+  ).then((entries) => Object.fromEntries(entries));
+}
 
-            const response = await res.text();
-
-            if (!response) {
-              return resolve(null);
-            }
-
-            resolve(response as ResBody);
+function createApi(_url: string) {
+  const headers: Record<string, string | Promise<string | null>> = {
+    'Content-Type': 'application/json',
+  };
+  return {
+    headers,
+    async fetch<ReqBody, ResBody>(
+      path: string,
+      data: ReqBody,
+      options?: RequestInit
+    ): Promise<ResBody | null> {
+      const url = `${_url}${path}`;
+      let timer: ReturnType<typeof setTimeout>;
+      const h = await awaitProperties(headers);
+      return new Promise((resolve) => {
+        const wrappedFetch = (attempt: number) => {
+          clearTimeout(timer);
+          fetch(url, {
+            headers: h,
+            method: 'POST',
+            body: JSON.stringify(data ?? {}),
+            keepalive: true,
+            ...(options ?? {}),
           })
-          .catch(() => {
-            return retry(attempt, resolve);
-          });
-      };
+            .then(async (res) => {
+              if (res.status !== 200 && res.status !== 202) {
+                return retry(attempt, resolve);
+              }
 
-      function retry(
-        attempt: number,
-        resolve: (value: ResBody | null) => void
-      ) {
-        if (attempt > 1) {
-          return resolve(null);
+              const response = await res.text();
+
+              if (!response) {
+                return resolve(null);
+              }
+
+              resolve(response as ResBody);
+            })
+            .catch(() => {
+              return retry(attempt, resolve);
+            });
+        };
+
+        function retry(
+          attempt: number,
+          resolve: (value: ResBody | null) => void
+        ) {
+          if (attempt > 1) {
+            return resolve(null);
+          }
+
+          timer = setTimeout(
+            () => {
+              wrappedFetch(attempt + 1);
+            },
+            Math.pow(2, attempt) * 500
+          );
         }
 
-        timer = setTimeout(
-          () => {
-            wrappedFetch(attempt + 1);
-          },
-          Math.pow(2, attempt) * 500
-        );
-      }
-
-      wrappedFetch(0);
-    });
+        wrappedFetch(0);
+      });
+    },
   };
 }
 
 export class Mixan<Options extends MixanOptions = MixanOptions> {
   public options: Options;
-  private api: ReturnType<typeof createApi>;
+  public api: ReturnType<typeof createApi>;
   private state: MixanState = {
     properties: {},
   };
 
   constructor(options: Options) {
     this.options = options;
-    this.api = createApi(options.url, options.clientId, options.clientSecret);
+    this.api = createApi(options.url);
+    this.api.headers['mixan-client-id'] = options.clientId;
+    if (this.options.clientSecret) {
+      this.api.headers['mixan-client-secret'] = this.options.clientSecret;
+    }
   }
 
   // Public
@@ -97,61 +116,49 @@ export class Mixan<Options extends MixanOptions = MixanOptions> {
     this.state.properties = properties ?? {};
   }
 
-  // public setUser(payload: Omit<BatchUpdateProfilePayload, 'profileId'>) {
-  //   this.batcher.add({
-  //     type: 'update_profile',
-  //     payload: {
-  //       ...payload,
-  //       properties: payload.properties ?? {},
-  //       profileId: this.state.profileId,
-  //     },
-  //   });
-  // }
-
-  // public increment(name: string, value: number) {
-  //   this.batcher.add({
-  //     type: 'increment',
-  //     payload: {
-  //       name,
-  //       value,
-  //       profileId: this.state.profileId,
-  //     },
-  //   });
-  // }
-
-  // public decrement(name: string, value: number) {
-  //   this.batcher.add({
-  //     type: 'decrement',
-  //     payload: {
-  //       name,
-  //       value,
-  //       profileId: this.state.profileId,
-  //     },
-  //   });
-  // }
-
-  private getProfileId() {
-    if (this.state.profileId) {
-      return this.state.profileId;
-    } else if (this.options.getProfileId) {
-      this.state.profileId = this.options.getProfileId() || undefined;
-    }
-  }
-
-  public async event(name: string, properties?: Record<string, unknown>) {
-    const profileId = await this.api<PostEventPayload, string>('/event', {
-      name,
+  public setUser(payload: Omit<UpdateProfilePayload, 'profileId'>) {
+    this.api.fetch<UpdateProfilePayload, string>('/profile', {
+      profileId: this.getProfileId(),
+      ...payload,
       properties: {
         ...this.state.properties,
-        ...(properties ?? {}),
+        ...payload.properties,
       },
-      timestamp: this.timestamp(),
+    });
+  }
+
+  public increment(property: string, value: number) {
+    this.api.fetch<IncrementProfilePayload, string>('/profile/increment', {
+      property,
+      value,
       profileId: this.getProfileId(),
     });
+  }
 
-    if (this.options.setProfileId && profileId) {
-      this.options.setProfileId(profileId);
-    }
+  public decrement(property: string, value: number) {
+    this.api.fetch<DecrementProfilePayload, string>('/profile/decrement', {
+      property,
+      value,
+      profileId: this.getProfileId(),
+    });
+  }
+
+  public event(name: string, properties?: Record<string, unknown>) {
+    this.api
+      .fetch<PostEventPayload, string>('/event', {
+        name,
+        properties: {
+          ...this.state.properties,
+          ...(properties ?? {}),
+        },
+        timestamp: this.timestamp(),
+        profileId: this.getProfileId(),
+      })
+      .then((profileId) => {
+        if (this.options.setProfileId && profileId) {
+          this.options.setProfileId(profileId);
+        }
+      });
   }
 
   public setGlobalProperties(properties: Record<string, unknown>) {
@@ -168,21 +175,17 @@ export class Mixan<Options extends MixanOptions = MixanOptions> {
     }
   }
 
-  public setUserProperty(name: string, value: unknown, update = true) {
-    // this.batcher.add({
-    //   type: 'set_profile_property',
-    //   payload: {
-    //     name,
-    //     value,
-    //     update,
-    //     profileId: this.state.profileId,
-    //   },
-    // });
-  }
-
   // Private
 
   private timestamp() {
     return new Date().toISOString();
   }
+
+  private getProfileId() {
+    if (this.state.profileId) {
+      return this.state.profileId;
+    } else if (this.options.getProfileId) {
+      this.state.profileId = this.options.getProfileId() || undefined;
+    }
+  }
 }

packages/types/index.ts

@@ -138,5 +138,31 @@ export interface PostEventPayload {
   name: string;
   timestamp: string;
   profileId?: string;
-  properties?: Record<string, unknown>;
+  properties?: Record<string, unknown> & {
+    title?: string | undefined;
+    referrer?: string | undefined;
+    path?: string | undefined;
+  };
+}
+
+export interface UpdateProfilePayload {
+  profileId?: string;
+  id?: string;
+  first_name?: string;
+  last_name?: string;
+  email?: string;
+  avatar?: string;
+  properties?: MixanJson;
+}
+
+export interface IncrementProfilePayload {
+  profileId?: string;
+  property: string;
+  value: number;
+}
+
+export interface DecrementProfilePayload {
+  profileId?: string;
+  property: string;
+  value: number;
 }