improvements while testing

apps/api/src/controllers/track.controller.ts

@@ -7,7 +7,9 @@ import { assocPath, pathOr } from 'ramda';
 import { generateDeviceId } from '@openpanel/common';
 import {
   ch,
+  chQuery,
   getProfileById,
+  getProfileId,
   getSalts,
   TABLE_NAMES,
   upsertProfile,
@@ -31,6 +33,23 @@ export async function handler(
   const ip = getClientIp(request)!;
   const ua = request.headers['user-agent']!;
   const projectId = request.client?.projectId;
+  const profileId =
+    projectId && request.body.payload.profileId
+      ? await getProfileId({
+          projectId,
+          profileId: request.body.payload.profileId,
+        })
+      : undefined;
+
+  if (profileId) {
+    request.body.payload.profileId = profileId;
+  }
+
+  console.log(
+    '> Request',
+    request.body.type,
+    JSON.stringify(request.body.payload, null, 2)
+  );
 
   if (!projectId) {
     reply.status(400).send('missing origin');

apps/docs/src/pages/docs/script.mdx

@@ -176,8 +176,18 @@ declare global {
 
 #### Strict typing (from sdk)
 
+<Steps>
+##### Step 1: Install the SDK
+
+```bash
+npm install @openpanel/web
+```
+
+##### Step 2: Create a type definition file
+
 Create a `op.d.ts`file and paste the following code:
 
 ```ts filename="op.d.ts"
 /// <reference types="@openpanel/web" />
 ```
+</Steps>

apps/public/public/op1.js

@@ -0,0 +1,2 @@
+"use strict";(()=>{var d=class{constructor(e){this.baseUrl=e.baseUrl,this.headers={"Content-Type":"application/json",...e.defaultHeaders},this.maxRetries=e.maxRetries??3,this.initialRetryDelay=e.initialRetryDelay??500}async resolveHeaders(){let e={};for(let[r,i]of Object.entries(this.headers)){let t=await i;t!==null&&(e[r]=t)}return e}addHeader(e,r){this.headers[e]=r}async post(e,r,i,t){try{let s=await fetch(e,{method:"POST",headers:await this.resolveHeaders(),body:JSON.stringify(r??{}),keepalive:!0,...i});if(s.status===401)return null;if(s.status!==200&&s.status!==202)throw new Error(`HTTP error! status: ${s.status}`);let n=await s.text();return n?JSON.parse(n):null}catch(s){if(t<this.maxRetries){let n=this.initialRetryDelay*Math.pow(2,t);return await new Promise(a=>setTimeout(a,n)),this.post(e,r,i,t+1)}return console.error("Max retries reached:",s),null}}async fetch(e,r,i={}){let t=`${this.baseUrl}${e}`;return this.post(t,r,i,0)}},c=class{constructor(e){this.options=e,this.queue=[];let r={"openpanel-client-id":e.clientId};e.clientSecret&&(r["openpanel-client-secret"]=e.clientSecret),r["openpanel-sdk"]=e.sdk||"node",r["openpanel-sdk-version"]=e.sdkVersion||"1.0.0-beta",this.api=new d({baseUrl:e.apiUrl||"https://api.openpanel.dev",defaultHeaders:r})}init(){}ready(){this.options.waitForProfile=!1,this.flush()}async send(e){return this.options.disable||this.options.filter&&!this.options.filter(e)?Promise.resolve():this.options.waitForProfile&&!this.profileId?(this.queue.push(e),Promise.resolve()):this.api.fetch("/track",e)}setGlobalProperties(e){this.global={...this.global,...e}}async track(e,r){return this.send({type:"track",payload:{name:e,profileId:r?.profileId??this.profileId,properties:{...this.global??{},...r??{}}}})}async identify(e){if(e.profileId&&(this.profileId=e.profileId,this.flush()),Object.keys(e).length>1)return this.send({type:"identify",payload:{...e,properties:{...this.global,...e.properties}}})}async alias(e){return this.send({type:"alias",payload:e})}async increment(e){return this.send({type:"increment",payload:e})}async decrement(e){return this.send({type:"decrement",payload:e})}clear(){this.profileId=void 0}flush(){this.queue.forEach(e=>{this.send({...e,payload:{...e.payload,profileId:e.payload.profileId??this.profileId}})}),this.queue=[]}};function u(e){return e.replace(/([-_][a-z])/gi,r=>r.toUpperCase().replace("-","").replace("_",""))}var p=class extends c{constructor(i){super({...i,sdk:"web",sdkVersion:"1.0.0-beta"});this.options=i;this.lastPath="";this.isServer()||(this.setGlobalProperties({__referrer:document.referrer}),this.options.trackScreenViews&&this.trackScreenViews(),this.options.trackOutgoingLinks&&this.trackOutgoingLinks(),this.options.trackAttributes&&this.trackAttributes())}debounce(i,t){clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(i,t)}isServer(){return typeof document>"u"}trackOutgoingLinks(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("a");if(s&&t){let n=s.getAttribute("href");n?.startsWith("http")&&super.track("link_out",{href:n,text:s.innerText||s.getAttribute("title")||t.getAttribute("alt")||t.getAttribute("title")})}})}trackScreenViews(){if(this.isServer())return;this.screenView();let i=history.pushState;history.pushState=function(...a){let o=i.apply(this,a);return window.dispatchEvent(new Event("pushstate")),window.dispatchEvent(new Event("locationchange")),o};let t=history.replaceState;history.replaceState=function(...a){let o=t.apply(this,a);return window.dispatchEvent(new Event("replacestate")),window.dispatchEvent(new Event("locationchange")),o},window.addEventListener("popstate",function(){window.dispatchEvent(new Event("locationchange"))});let s=()=>this.debounce(()=>this.screenView(),50);this.options.trackHashChanges?window.addEventListener("hashchange",s):window.addEventListener("locationchange",s)}trackAttributes(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("button"),n=t.closest("a"),a=s?.getAttribute("data-track")?s:n?.getAttribute("data-track")?n:null;if(a){let o={};for(let l of a.attributes)l.name.startsWith("data-")&&l.name!=="data-track"&&(o[u(l.name.replace(/^data-/,""))]=l.value);let h=a.getAttribute("data-track");h&&super.track(h,o)}})}screenView(i,t){if(this.isServer())return;let s,n;typeof i=="string"?(s=i,n=t):(s=window.location.href,n=i),this.lastPath!==s&&(this.lastPath=s,super.track("screen_view",{...n??{},__path:s,__title:document.title}))}};(e=>{if(e.op&&"q"in e.op){let r=e.op.q||[],i=new p(r.shift()[1]);r.forEach(t=>{t[0]in i&&i[t[0]](...t.slice(1))}),e.op=(t,...s)=>{let n=i[t]?i[t].bind(i):void 0;typeof n=="function"?n(...s):console.warn(`OpenPanel: ${t} is not a function`)},e.openpanel=i}})(window);})();
+//# sourceMappingURL=tracker.global.js.map

apps/public/public/tracker.js

@@ -1,2 +0,0 @@
-"use strict";(()=>{var d=class{constructor(e){this.baseUrl=e.baseUrl,this.headers={"Content-Type":"application/json",...e.defaultHeaders},this.maxRetries=e.maxRetries??3,this.initialRetryDelay=e.initialRetryDelay??500}async resolveHeaders(){let e={};for(let[r,i]of Object.entries(this.headers)){let t=await i;t!==null&&(e[r]=t)}return e}addHeader(e,r){this.headers[e]=r}async post(e,r,i,t){try{let s=await fetch(e,{method:"POST",headers:await this.resolveHeaders(),body:JSON.stringify(r??{}),keepalive:!0,...i});if(s.status===401)return null;if(s.status!==200&&s.status!==202)throw new Error(`HTTP error! status: ${s.status}`);let n=await s.text();return n?JSON.parse(n):null}catch(s){if(t<this.maxRetries){let n=this.initialRetryDelay*Math.pow(2,t);return await new Promise(a=>setTimeout(a,n)),this.post(e,r,i,t+1)}return console.error("Max retries reached:",s),null}}async fetch(e,r,i={}){let t=`${this.baseUrl}${e}`;return this.post(t,r,i,0)}},h=class{constructor(e){this.options=e,this.queue=[];let r={"openpanel-client-id":e.clientId};e.clientSecret&&(r["openpanel-client-secret"]=e.clientSecret),r["openpanel-sdk"]=e.sdk||"node",r["openpanel-sdk-version"]=e.sdkVersion||"0.0.11-beta",this.api=new d({baseUrl:e.apiUrl||"https://api.openpanel.dev",defaultHeaders:r})}init(){}ready(){this.options.waitForProfile=!1,this.flush()}async send(e){return this.options.filter&&!this.options.filter(e)?Promise.resolve():this.options.waitForProfile&&!this.profileId?(this.queue.push(e),Promise.resolve()):this.api.fetch("/track",e)}setGlobalProperties(e){this.global={...this.global,...e}}async track(e,r){return this.send({type:"track",payload:{name:e,profileId:r?.profileId??this.profileId,properties:{...this.global??{},...r??{}}}})}async identify(e){if(e.profileId&&(this.profileId=e.profileId,this.flush()),Object.keys(e).length>1)return this.send({type:"identify",payload:{...e,properties:{...this.global,...e.properties}}})}async alias(e){return this.send({type:"alias",payload:e})}async increment(e){return this.send({type:"increment",payload:e})}async decrement(e){return this.send({type:"decrement",payload:e})}clear(){this.profileId=void 0}flush(){this.queue.forEach(e=>{this.send({...e,payload:{...e.payload,profileId:this.profileId}})}),this.queue=[]}};function u(e){return e.replace(/([-_][a-z])/gi,r=>r.toUpperCase().replace("-","").replace("_",""))}var c=class extends h{constructor(i){super({...i,sdk:"web",sdkVersion:"0.0.11-beta"});this.options=i;this.lastPath="";this.isServer()||(this.setGlobalProperties({__referrer:document.referrer}),this.options.trackScreenViews&&this.trackScreenViews(),this.options.trackOutgoingLinks&&this.trackOutgoingLinks(),this.options.trackAttributes&&this.trackAttributes())}debounce(i,t){clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(i,t)}isServer(){return typeof document>"u"}trackOutgoingLinks(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("a");if(s&&t){let n=s.getAttribute("href");n?.startsWith("http")&&super.track("link_out",{href:n,text:s.innerText||s.getAttribute("title")||t.getAttribute("alt")||t.getAttribute("title")})}})}trackScreenViews(){if(this.isServer())return;this.screenView();let i=history.pushState;history.pushState=function(...a){let o=i.apply(this,a);return window.dispatchEvent(new Event("pushstate")),window.dispatchEvent(new Event("locationchange")),o};let t=history.replaceState;history.replaceState=function(...a){let o=t.apply(this,a);return window.dispatchEvent(new Event("replacestate")),window.dispatchEvent(new Event("locationchange")),o},window.addEventListener("popstate",function(){window.dispatchEvent(new Event("locationchange"))});let s=()=>this.debounce(()=>this.screenView(),50);this.options.trackHashChanges?window.addEventListener("hashchange",s):window.addEventListener("locationchange",s)}trackAttributes(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("button"),n=t.closest("a"),a=s?.getAttribute("data-event")?s:n?.getAttribute("data-event")?n:null;if(a){let o={};for(let l of a.attributes)l.name.startsWith("data-")&&l.name!=="data-event"&&(o[u(l.name.replace(/^data-/,""))]=l.value);let p=a.getAttribute("data-event");p&&super.track(p,o)}})}screenView(i,t){if(this.isServer())return;let s,n;typeof i=="string"?(s=i,n=t):(s=window.location.href,n=i),this.lastPath!==s&&(this.lastPath=s,super.track("screen_view",{...n??{},__path:s,__title:document.title}))}};(e=>{if(e.op&&"q"in e.op){let r=e.op.q||[],i=new c(r.shift()[1]);r.forEach(t=>{t[0]in i&&i[t[0]](...t.slice(1))}),e.op=(t,...s)=>{let n=i[t]?i[t].bind(i):void 0;typeof n=="function"?n(...s):console.warn(`op.js: ${t} is not a function`)},e.openpanel=i}})(window);})();
-//# sourceMappingURL=tracker.global.js.map

apps/worker/src/jobs/events.incoming-event.ts

@@ -18,7 +18,7 @@ import type {
   EventsQueuePayloadCreateSessionEnd,
   EventsQueuePayloadIncomingEvent,
 } from '@openpanel/queue';
-import { getRedisQueue } from '@openpanel/redis';
+import { cacheable, getRedisQueue } from '@openpanel/redis';
 
 const GLOBAL_PROPERTIES = ['__path', '__referrer'];
 const SESSION_TIMEOUT = 1000 * 60 * 30;
@@ -47,10 +47,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {
   };
 
   // this will get the profileId from the alias table if it exists
-  const profileId = await getProfileId({
-    projectId,
-    profileId: body.profileId,
-  });
+  const profileId = body.profileId ?? '';
   const createdAt = new Date(body.timestamp);
   const url = getProperty('__path');
   const { path, hash, query, origin } = parsePath(url);
@@ -259,29 +256,3 @@ async function getSessionEnd({
   // Create session
   return null;
 }
-
-async function getProfileId({
-  profileId,
-  projectId,
-}: {
-  profileId: string | undefined;
-  projectId: string;
-}) {
-  if (!profileId) {
-    return '';
-  }
-
-  const res = await chQuery<{
-    alias: string;
-    profile_id: string;
-    project_id: string;
-  }>(
-    `SELECT * FROM ${TABLE_NAMES.alias} WHERE project_id = '${projectId}' AND (alias = '${profileId}' OR profile_id = '${profileId}')`
-  );
-
-  if (res[0]) {
-    return res[0].profile_id;
-  }
-
-  return profileId;
-}

packages/db/clickhouse_init.sql

@@ -60,6 +60,15 @@ CREATE TABLE IF NOT EXISTS openpanel.profiles (
 ORDER BY
   (id) SETTINGS index_granularity = 8192;
 
+CREATE TABLE IF NOT EXISTS openpanel.profile_aliases (
+  `project_id` String,
+  `profile_id` String,
+  `alias` String,
+  `created_at` DateTime
+) ENGINE = MergeTree
+ORDER BY
+  (project_id, profile_id, alias) SETTINGS index_granularity = 8192;
+
 --- Materialized views (DAU)
 CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMMDD(date)
 ORDER BY

packages/db/src/clickhouse-client.ts

@@ -111,10 +111,13 @@ export async function chQuery<T extends Record<string, any>>(
   return (await chQueryWithMeta<T>(query)).data;
 }
 
-export function formatClickhouseDate(_date: Date | string, skipTime = false) {
+export function formatClickhouseDate(
+  _date: Date | string,
+  skipTime = false
+): string {
   const date = typeof _date === 'string' ? new Date(_date) : _date;
   if (skipTime) {
-    return date.toISOString().split('T')[0];
+    return date.toISOString().split('T')[0]!;
   }
   return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
 }

packages/db/src/services/profile.service.ts

@@ -1,6 +1,7 @@
 import { escape } from 'sqlstring';
 
 import { toObject } from '@openpanel/common';
+import { cacheable } from '@openpanel/redis';
 import type { IChartEventFilter } from '@openpanel/validation';
 
 import { profileBuffer } from '../buffers';
@@ -191,3 +192,31 @@ export async function upsertProfile({
     is_external: isExternal,
   });
 }
+
+export async function getProfileId({
+  profileId,
+  projectId,
+}: {
+  profileId: string | undefined;
+  projectId: string;
+}) {
+  if (!profileId) {
+    return '';
+  }
+
+  const res = await chQuery<{
+    alias: string;
+    profile_id: string;
+    project_id: string;
+  }>(
+    `SELECT * FROM ${TABLE_NAMES.alias} WHERE project_id = '${projectId}' AND (alias = '${profileId}' OR profile_id = '${profileId}')`
+  );
+
+  if (res[0]) {
+    return res[0].profile_id;
+  }
+
+  return profileId;
+}
+
+export const getProfileIdCached = cacheable(getProfileId, 60 * 30);

packages/redis/cachable.ts

@@ -2,7 +2,7 @@ import { getRedisCache } from './redis';
 
 export function cacheable<T extends (...args: any) => any>(
   fn: T,
-  expire: number
+  expireInSec: number
 ) {
   return async function (
     ...args: Parameters<T>
@@ -20,7 +20,7 @@ export function cacheable<T extends (...args: any) => any>(
     const result = await fn(...(args as any));
 
     if (result !== undefined || result !== null) {
-      getRedisCache().setex(key, expire, JSON.stringify(result));
+      getRedisCache().setex(key, expireInSec, JSON.stringify(result));
     }
 
     return result;

packages/sdks/sdk/src/index.ts

@@ -67,7 +67,7 @@ export type OpenPanelOptions = {
   sdkVersion?: string;
   waitForProfile?: boolean;
   filter?: (payload: TrackHandlerPayload) => boolean;
-  disable?: boolean;
+  disabled?: boolean;
 };
 
 export class OpenPanel {
@@ -106,7 +106,7 @@ export class OpenPanel {
   }
 
   async send(payload: TrackHandlerPayload) {
-    if (this.options.disable) {
+    if (this.options.disabled) {
       return Promise.resolve();
     }