diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts index 634fd054..1123a3e3 100644 --- a/apps/api/src/controllers/track.controller.ts +++ b/apps/api/src/controllers/track.controller.ts @@ -7,7 +7,9 @@ import { assocPath, pathOr } from 'ramda'; import { generateDeviceId } from '@openpanel/common'; import { ch, + chQuery, getProfileById, + getProfileId, getSalts, TABLE_NAMES, upsertProfile, @@ -31,6 +33,23 @@ export async function handler( const ip = getClientIp(request)!; const ua = request.headers['user-agent']!; const projectId = request.client?.projectId; + const profileId = + projectId && request.body.payload.profileId + ? await getProfileId({ + projectId, + profileId: request.body.payload.profileId, + }) + : undefined; + + if (profileId) { + request.body.payload.profileId = profileId; + } + + console.log( + '> Request', + request.body.type, + JSON.stringify(request.body.payload, null, 2) + ); if (!projectId) { reply.status(400).send('missing origin'); diff --git a/apps/docs/src/pages/docs/script.mdx b/apps/docs/src/pages/docs/script.mdx index e11f651b..c706b6ef 100644 --- a/apps/docs/src/pages/docs/script.mdx +++ b/apps/docs/src/pages/docs/script.mdx @@ -176,8 +176,18 @@ declare global { #### Strict typing (from sdk) + +##### Step 1: Install the SDK + +```bash +npm install @openpanel/web +``` + +##### Step 2: Create a type definition file + Create a `op.d.ts`file and paste the following code: ```ts filename="op.d.ts" /// -``` \ No newline at end of file +``` + \ No newline at end of file diff --git a/apps/public/public/op1.js b/apps/public/public/op1.js new file mode 100644 index 00000000..fc4eee36 --- /dev/null +++ b/apps/public/public/op1.js @@ -0,0 +1,2 @@ +"use strict";(()=>{var d=class{constructor(e){this.baseUrl=e.baseUrl,this.headers={"Content-Type":"application/json",...e.defaultHeaders},this.maxRetries=e.maxRetries??3,this.initialRetryDelay=e.initialRetryDelay??500}async resolveHeaders(){let e={};for(let[r,i]of Object.entries(this.headers)){let t=await i;t!==null&&(e[r]=t)}return e}addHeader(e,r){this.headers[e]=r}async post(e,r,i,t){try{let s=await fetch(e,{method:"POST",headers:await this.resolveHeaders(),body:JSON.stringify(r??{}),keepalive:!0,...i});if(s.status===401)return null;if(s.status!==200&&s.status!==202)throw new Error(`HTTP error! status: ${s.status}`);let n=await s.text();return n?JSON.parse(n):null}catch(s){if(tsetTimeout(a,n)),this.post(e,r,i,t+1)}return console.error("Max retries reached:",s),null}}async fetch(e,r,i={}){let t=`${this.baseUrl}${e}`;return this.post(t,r,i,0)}},c=class{constructor(e){this.options=e,this.queue=[];let r={"openpanel-client-id":e.clientId};e.clientSecret&&(r["openpanel-client-secret"]=e.clientSecret),r["openpanel-sdk"]=e.sdk||"node",r["openpanel-sdk-version"]=e.sdkVersion||"1.0.0-beta",this.api=new d({baseUrl:e.apiUrl||"https://api.openpanel.dev",defaultHeaders:r})}init(){}ready(){this.options.waitForProfile=!1,this.flush()}async send(e){return this.options.disable||this.options.filter&&!this.options.filter(e)?Promise.resolve():this.options.waitForProfile&&!this.profileId?(this.queue.push(e),Promise.resolve()):this.api.fetch("/track",e)}setGlobalProperties(e){this.global={...this.global,...e}}async track(e,r){return this.send({type:"track",payload:{name:e,profileId:r?.profileId??this.profileId,properties:{...this.global??{},...r??{}}}})}async identify(e){if(e.profileId&&(this.profileId=e.profileId,this.flush()),Object.keys(e).length>1)return this.send({type:"identify",payload:{...e,properties:{...this.global,...e.properties}}})}async alias(e){return this.send({type:"alias",payload:e})}async increment(e){return this.send({type:"increment",payload:e})}async decrement(e){return this.send({type:"decrement",payload:e})}clear(){this.profileId=void 0}flush(){this.queue.forEach(e=>{this.send({...e,payload:{...e.payload,profileId:e.payload.profileId??this.profileId}})}),this.queue=[]}};function u(e){return e.replace(/([-_][a-z])/gi,r=>r.toUpperCase().replace("-","").replace("_",""))}var p=class extends c{constructor(i){super({...i,sdk:"web",sdkVersion:"1.0.0-beta"});this.options=i;this.lastPath="";this.isServer()||(this.setGlobalProperties({__referrer:document.referrer}),this.options.trackScreenViews&&this.trackScreenViews(),this.options.trackOutgoingLinks&&this.trackOutgoingLinks(),this.options.trackAttributes&&this.trackAttributes())}debounce(i,t){clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(i,t)}isServer(){return typeof document>"u"}trackOutgoingLinks(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("a");if(s&&t){let n=s.getAttribute("href");n?.startsWith("http")&&super.track("link_out",{href:n,text:s.innerText||s.getAttribute("title")||t.getAttribute("alt")||t.getAttribute("title")})}})}trackScreenViews(){if(this.isServer())return;this.screenView();let i=history.pushState;history.pushState=function(...a){let o=i.apply(this,a);return window.dispatchEvent(new Event("pushstate")),window.dispatchEvent(new Event("locationchange")),o};let t=history.replaceState;history.replaceState=function(...a){let o=t.apply(this,a);return window.dispatchEvent(new Event("replacestate")),window.dispatchEvent(new Event("locationchange")),o},window.addEventListener("popstate",function(){window.dispatchEvent(new Event("locationchange"))});let s=()=>this.debounce(()=>this.screenView(),50);this.options.trackHashChanges?window.addEventListener("hashchange",s):window.addEventListener("locationchange",s)}trackAttributes(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("button"),n=t.closest("a"),a=s?.getAttribute("data-track")?s:n?.getAttribute("data-track")?n:null;if(a){let o={};for(let l of a.attributes)l.name.startsWith("data-")&&l.name!=="data-track"&&(o[u(l.name.replace(/^data-/,""))]=l.value);let h=a.getAttribute("data-track");h&&super.track(h,o)}})}screenView(i,t){if(this.isServer())return;let s,n;typeof i=="string"?(s=i,n=t):(s=window.location.href,n=i),this.lastPath!==s&&(this.lastPath=s,super.track("screen_view",{...n??{},__path:s,__title:document.title}))}};(e=>{if(e.op&&"q"in e.op){let r=e.op.q||[],i=new p(r.shift()[1]);r.forEach(t=>{t[0]in i&&i[t[0]](...t.slice(1))}),e.op=(t,...s)=>{let n=i[t]?i[t].bind(i):void 0;typeof n=="function"?n(...s):console.warn(`OpenPanel: ${t} is not a function`)},e.openpanel=i}})(window);})(); +//# sourceMappingURL=tracker.global.js.map \ No newline at end of file diff --git a/apps/public/public/tracker.js b/apps/public/public/tracker.js deleted file mode 100644 index 070a2779..00000000 --- a/apps/public/public/tracker.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict";(()=>{var d=class{constructor(e){this.baseUrl=e.baseUrl,this.headers={"Content-Type":"application/json",...e.defaultHeaders},this.maxRetries=e.maxRetries??3,this.initialRetryDelay=e.initialRetryDelay??500}async resolveHeaders(){let e={};for(let[r,i]of Object.entries(this.headers)){let t=await i;t!==null&&(e[r]=t)}return e}addHeader(e,r){this.headers[e]=r}async post(e,r,i,t){try{let s=await fetch(e,{method:"POST",headers:await this.resolveHeaders(),body:JSON.stringify(r??{}),keepalive:!0,...i});if(s.status===401)return null;if(s.status!==200&&s.status!==202)throw new Error(`HTTP error! status: ${s.status}`);let n=await s.text();return n?JSON.parse(n):null}catch(s){if(tsetTimeout(a,n)),this.post(e,r,i,t+1)}return console.error("Max retries reached:",s),null}}async fetch(e,r,i={}){let t=`${this.baseUrl}${e}`;return this.post(t,r,i,0)}},h=class{constructor(e){this.options=e,this.queue=[];let r={"openpanel-client-id":e.clientId};e.clientSecret&&(r["openpanel-client-secret"]=e.clientSecret),r["openpanel-sdk"]=e.sdk||"node",r["openpanel-sdk-version"]=e.sdkVersion||"0.0.11-beta",this.api=new d({baseUrl:e.apiUrl||"https://api.openpanel.dev",defaultHeaders:r})}init(){}ready(){this.options.waitForProfile=!1,this.flush()}async send(e){return this.options.filter&&!this.options.filter(e)?Promise.resolve():this.options.waitForProfile&&!this.profileId?(this.queue.push(e),Promise.resolve()):this.api.fetch("/track",e)}setGlobalProperties(e){this.global={...this.global,...e}}async track(e,r){return this.send({type:"track",payload:{name:e,profileId:r?.profileId??this.profileId,properties:{...this.global??{},...r??{}}}})}async identify(e){if(e.profileId&&(this.profileId=e.profileId,this.flush()),Object.keys(e).length>1)return this.send({type:"identify",payload:{...e,properties:{...this.global,...e.properties}}})}async alias(e){return this.send({type:"alias",payload:e})}async increment(e){return this.send({type:"increment",payload:e})}async decrement(e){return this.send({type:"decrement",payload:e})}clear(){this.profileId=void 0}flush(){this.queue.forEach(e=>{this.send({...e,payload:{...e.payload,profileId:this.profileId}})}),this.queue=[]}};function u(e){return e.replace(/([-_][a-z])/gi,r=>r.toUpperCase().replace("-","").replace("_",""))}var c=class extends h{constructor(i){super({...i,sdk:"web",sdkVersion:"0.0.11-beta"});this.options=i;this.lastPath="";this.isServer()||(this.setGlobalProperties({__referrer:document.referrer}),this.options.trackScreenViews&&this.trackScreenViews(),this.options.trackOutgoingLinks&&this.trackOutgoingLinks(),this.options.trackAttributes&&this.trackAttributes())}debounce(i,t){clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(i,t)}isServer(){return typeof document>"u"}trackOutgoingLinks(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("a");if(s&&t){let n=s.getAttribute("href");n?.startsWith("http")&&super.track("link_out",{href:n,text:s.innerText||s.getAttribute("title")||t.getAttribute("alt")||t.getAttribute("title")})}})}trackScreenViews(){if(this.isServer())return;this.screenView();let i=history.pushState;history.pushState=function(...a){let o=i.apply(this,a);return window.dispatchEvent(new Event("pushstate")),window.dispatchEvent(new Event("locationchange")),o};let t=history.replaceState;history.replaceState=function(...a){let o=t.apply(this,a);return window.dispatchEvent(new Event("replacestate")),window.dispatchEvent(new Event("locationchange")),o},window.addEventListener("popstate",function(){window.dispatchEvent(new Event("locationchange"))});let s=()=>this.debounce(()=>this.screenView(),50);this.options.trackHashChanges?window.addEventListener("hashchange",s):window.addEventListener("locationchange",s)}trackAttributes(){this.isServer()||document.addEventListener("click",i=>{let t=i.target,s=t.closest("button"),n=t.closest("a"),a=s?.getAttribute("data-event")?s:n?.getAttribute("data-event")?n:null;if(a){let o={};for(let l of a.attributes)l.name.startsWith("data-")&&l.name!=="data-event"&&(o[u(l.name.replace(/^data-/,""))]=l.value);let p=a.getAttribute("data-event");p&&super.track(p,o)}})}screenView(i,t){if(this.isServer())return;let s,n;typeof i=="string"?(s=i,n=t):(s=window.location.href,n=i),this.lastPath!==s&&(this.lastPath=s,super.track("screen_view",{...n??{},__path:s,__title:document.title}))}};(e=>{if(e.op&&"q"in e.op){let r=e.op.q||[],i=new c(r.shift()[1]);r.forEach(t=>{t[0]in i&&i[t[0]](...t.slice(1))}),e.op=(t,...s)=>{let n=i[t]?i[t].bind(i):void 0;typeof n=="function"?n(...s):console.warn(`op.js: ${t} is not a function`)},e.openpanel=i}})(window);})(); -//# sourceMappingURL=tracker.global.js.map \ No newline at end of file diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts index 100db59f..bf2194cf 100644 --- a/apps/worker/src/jobs/events.incoming-event.ts +++ b/apps/worker/src/jobs/events.incoming-event.ts @@ -18,7 +18,7 @@ import type { EventsQueuePayloadCreateSessionEnd, EventsQueuePayloadIncomingEvent, } from '@openpanel/queue'; -import { getRedisQueue } from '@openpanel/redis'; +import { cacheable, getRedisQueue } from '@openpanel/redis'; const GLOBAL_PROPERTIES = ['__path', '__referrer']; const SESSION_TIMEOUT = 1000 * 60 * 30; @@ -47,10 +47,7 @@ export async function incomingEvent(job: Job) { }; // this will get the profileId from the alias table if it exists - const profileId = await getProfileId({ - projectId, - profileId: body.profileId, - }); + const profileId = body.profileId ?? ''; const createdAt = new Date(body.timestamp); const url = getProperty('__path'); const { path, hash, query, origin } = parsePath(url); @@ -259,29 +256,3 @@ async function getSessionEnd({ // Create session return null; } - -async function getProfileId({ - profileId, - projectId, -}: { - profileId: string | undefined; - projectId: string; -}) { - if (!profileId) { - return ''; - } - - const res = await chQuery<{ - alias: string; - profile_id: string; - project_id: string; - }>( - `SELECT * FROM ${TABLE_NAMES.alias} WHERE project_id = '${projectId}' AND (alias = '${profileId}' OR profile_id = '${profileId}')` - ); - - if (res[0]) { - return res[0].profile_id; - } - - return profileId; -} diff --git a/packages/db/clickhouse_init.sql b/packages/db/clickhouse_init.sql index 80075ee8..69d9ddb4 100644 --- a/packages/db/clickhouse_init.sql +++ b/packages/db/clickhouse_init.sql @@ -60,6 +60,15 @@ CREATE TABLE IF NOT EXISTS openpanel.profiles ( ORDER BY (id) SETTINGS index_granularity = 8192; +CREATE TABLE IF NOT EXISTS openpanel.profile_aliases ( + `project_id` String, + `profile_id` String, + `alias` String, + `created_at` DateTime +) ENGINE = MergeTree +ORDER BY + (project_id, profile_id, alias) SETTINGS index_granularity = 8192; + --- Materialized views (DAU) CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMMDD(date) ORDER BY diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.ts index 828c2da9..1184d7eb 100644 --- a/packages/db/src/clickhouse-client.ts +++ b/packages/db/src/clickhouse-client.ts @@ -111,10 +111,13 @@ export async function chQuery>( return (await chQueryWithMeta(query)).data; } -export function formatClickhouseDate(_date: Date | string, skipTime = false) { +export function formatClickhouseDate( + _date: Date | string, + skipTime = false +): string { const date = typeof _date === 'string' ? new Date(_date) : _date; if (skipTime) { - return date.toISOString().split('T')[0]; + return date.toISOString().split('T')[0]!; } return date.toISOString().replace('T', ' ').replace(/Z+$/, ''); } diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.ts index 74f2d6e9..4f2b43cb 100644 --- a/packages/db/src/services/profile.service.ts +++ b/packages/db/src/services/profile.service.ts @@ -1,6 +1,7 @@ import { escape } from 'sqlstring'; import { toObject } from '@openpanel/common'; +import { cacheable } from '@openpanel/redis'; import type { IChartEventFilter } from '@openpanel/validation'; import { profileBuffer } from '../buffers'; @@ -191,3 +192,31 @@ export async function upsertProfile({ is_external: isExternal, }); } + +export async function getProfileId({ + profileId, + projectId, +}: { + profileId: string | undefined; + projectId: string; +}) { + if (!profileId) { + return ''; + } + + const res = await chQuery<{ + alias: string; + profile_id: string; + project_id: string; + }>( + `SELECT * FROM ${TABLE_NAMES.alias} WHERE project_id = '${projectId}' AND (alias = '${profileId}' OR profile_id = '${profileId}')` + ); + + if (res[0]) { + return res[0].profile_id; + } + + return profileId; +} + +export const getProfileIdCached = cacheable(getProfileId, 60 * 30); diff --git a/packages/redis/cachable.ts b/packages/redis/cachable.ts index 4aa663dd..f2c03429 100644 --- a/packages/redis/cachable.ts +++ b/packages/redis/cachable.ts @@ -2,7 +2,7 @@ import { getRedisCache } from './redis'; export function cacheable any>( fn: T, - expire: number + expireInSec: number ) { return async function ( ...args: Parameters @@ -20,7 +20,7 @@ export function cacheable any>( const result = await fn(...(args as any)); if (result !== undefined || result !== null) { - getRedisCache().setex(key, expire, JSON.stringify(result)); + getRedisCache().setex(key, expireInSec, JSON.stringify(result)); } return result; diff --git a/packages/sdks/sdk/src/index.ts b/packages/sdks/sdk/src/index.ts index 66df67e0..12d318ee 100644 --- a/packages/sdks/sdk/src/index.ts +++ b/packages/sdks/sdk/src/index.ts @@ -67,7 +67,7 @@ export type OpenPanelOptions = { sdkVersion?: string; waitForProfile?: boolean; filter?: (payload: TrackHandlerPayload) => boolean; - disable?: boolean; + disabled?: boolean; }; export class OpenPanel { @@ -106,7 +106,7 @@ export class OpenPanel { } async send(payload: TrackHandlerPayload) { - if (this.options.disable) { + if (this.options.disabled) { return Promise.resolve(); }