batching events

packages/common/src/object.ts

@@ -13,9 +13,14 @@ export function toDots(
       };
     }
 
+    if (value === undefined || value === null) {
+      return acc;
+    }
+
     return {
       ...acc,
-      [`${path}${key}`]: typeof value === 'string' ? value.trim() : value,
+      [`${path}${key}`]:
+        typeof value === 'string' ? value.trim() : String(value),
     };
   }, {});
 }
@@ -52,3 +57,46 @@ export function getSuperJson<T>(str: string): T | null {
   }
   return json;
 }
+
+type AnyObject = Record<string, any>;
+
+export function deepMergeObjects<T>(target: AnyObject, source: AnyObject): T {
+  const merged: AnyObject = {};
+  // Include all keys from both objects
+  const allKeys = new Set([...Object.keys(target), ...Object.keys(source)]);
+
+  allKeys.forEach((key) => {
+    const targetValue = target[key];
+    const sourceValue = source[key];
+
+    if (
+      (isNil(sourceValue) && !isNil(targetValue)) ||
+      (sourceValue === '' &&
+        typeof targetValue === 'string' &&
+        targetValue !== '')
+    ) {
+      // Keep target value if source value is null or undefined
+      merged[key] = targetValue;
+    } else if (
+      sourceValue !== undefined &&
+      isObject(targetValue) &&
+      isObject(sourceValue)
+    ) {
+      // Recursively merge objects
+      merged[key] = deepMergeObjects(targetValue, sourceValue);
+    } else if (sourceValue !== undefined) {
+      // Directly assign any non-undefined source values
+      merged[key] = sourceValue;
+    } else if (sourceValue === undefined && target[key] !== undefined) {
+      // Keep target value if source value is undefined
+      merged[key] = targetValue;
+    }
+  });
+
+  return merged as T;
+}
+
+// Helper function to check if a value is an object (but not null or an array)
+function isObject(value: any): boolean {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}

packages/db/clickhouse_init.sql

@@ -0,0 +1,73 @@
+CREATE DATABASE IF NOT EXISTS openpanel;
+
+CREATE TABLE IF NOT EXISTS openpanel.events (
+  `id` UUID DEFAULT generateUUIDv4(),
+  `name` String,
+  `device_id` String,
+  `profile_id` String,
+  `project_id` String,
+  `session_id` String,
+  `path` String,
+  `origin` String,
+  `referrer` String,
+  `referrer_name` String,
+  `referrer_type` String,
+  `duration` UInt64,
+  `properties` Map(String, String),
+  `created_at` DateTime64(3),
+  `country` String,
+  `city` String,
+  `region` String,
+  `longitude` Nullable(Float32),
+  `latitude` Nullable(Float32),
+  `os` String,
+  `os_version` String,
+  `browser` String,
+  `browser_version` String,
+  -- device: mobile/desktop/tablet
+  `device` String,
+  -- brand: (Samsung, OnePlus)
+  `brand` String,
+  -- model: (Samsung Galaxy, iPhone X)
+  `model` String
+) ENGINE MergeTree
+ORDER BY
+  (project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
+
+CREATE TABLE IF NOT EXISTS openpanel.events_bots (
+  `id` UUID DEFAULT generateUUIDv4(),
+  `project_id` String,
+  `name` String,
+  `type` String,
+  `path` String,
+  `created_at` DateTime64(3),
+) ENGINE MergeTree
+ORDER BY
+  (project_id, created_at) SETTINGS index_granularity = 8192;
+
+CREATE TABLE IF NOT EXISTS openpanel.profiles (
+  `id` String,
+  `first_name` String,
+  `last_name` String,
+  `email` String,
+  `avatar` String,
+  `properties` Map(String, String),
+  `project_id` String,
+  `created_at` DateTime
+) ENGINE = ReplacingMergeTree(created_at)
+ORDER BY
+  (id) SETTINGS index_granularity = 8192;
+
+--- Materialized views (DAU)
+CREATE MATERIALIZED VIEW IF NOT EXISTS dau_mv ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMMDD(date)
+ORDER BY
+  (project_id, date) POPULATE AS
+SELECT
+  toDate(created_at) as date,
+  uniqState(profile_id) as profile_id,
+  project_id
+FROM
+  events
+GROUP BY
+  date,
+  project_id;

packages/db/index.ts

@@ -15,3 +15,4 @@ export * from './src/services/user.service';
 export * from './src/services/reference.service';
 export * from './src/services/id.service';
 export * from './src/services/retention.service';
+export * from './src/buffers';

packages/db/package.json

@@ -13,7 +13,7 @@
   },
   "dependencies": {
     "@clerk/nextjs": "^5.0.2",
-    "@clickhouse/client": "^1.0.1",
+    "@clickhouse/client": "^1.2.0",
     "@openpanel/common": "workspace:*",
     "@openpanel/constants": "workspace:*",
     "@openpanel/redis": "workspace:*",

packages/db/src/buffers/buffer.ts

@@ -0,0 +1,137 @@
+import type { Redis } from '@openpanel/redis';
+
+export const DELETE = '__DELETE__';
+
+export type QueueItem<T> = {
+  event: T;
+  index: number;
+};
+
+export type OnInsert<T> = (data: T) => unknown;
+
+export type OnCompleted<T> =
+  | ((data: T[]) => Promise<unknown[]>)
+  | ((data: T[]) => unknown[]);
+
+export type ProcessQueue<T> = (data: QueueItem<T>[]) => Promise<number[]>;
+
+export type Find<T, R = unknown> = (
+  callback: (item: QueueItem<T>) => boolean
+) => Promise<R | null>;
+
+export type FindMany<T, R = unknown> = (
+  callback: (item: QueueItem<T>) => boolean
+) => Promise<R[]>;
+
+export abstract class RedisBuffer<T> {
+  // constructor
+  public prefix = 'op:buffer';
+  public table: string;
+  public batchSize?: number;
+  public redis: Redis;
+
+  // abstract methods
+  public abstract onInsert?: OnInsert<T>;
+  public abstract onCompleted?: OnCompleted<T>;
+  public abstract processQueue: ProcessQueue<T>;
+  public abstract find: Find<T, unknown>;
+  public abstract findMany: FindMany<T, unknown>;
+
+  constructor(options: { table: string; redis: Redis; batchSize?: number }) {
+    this.table = options.table;
+    this.redis = options.redis;
+    this.batchSize = options.batchSize;
+  }
+
+  public getKey(name?: string) {
+    const key = this.prefix + ':' + this.table;
+    if (name) {
+      return `${key}:${name}`;
+    }
+    return key;
+  }
+
+  public async insert(value: T) {
+    this.onInsert?.(value);
+    await this.redis.rpush(this.getKey(), JSON.stringify(value));
+
+    const length = await this.redis.llen(this.getKey());
+    if (this.batchSize && length >= this.batchSize) {
+      this.flush();
+    }
+  }
+
+  public async flush() {
+    try {
+      const queue = await this.getQueue(this.batchSize || -1);
+
+      if (queue.length === 0) {
+        return {
+          count: 0,
+          data: [],
+        };
+      }
+
+      try {
+        const indexes = await this.processQueue(queue);
+        await this.deleteIndexes(indexes);
+        const data = indexes
+          .map((index) => queue[index]?.event)
+          .filter((event): event is T => event !== null);
+
+        if (this.onCompleted) {
+          const res = await this.onCompleted(data);
+          return {
+            count: res.length,
+            data: res,
+          };
+        }
+
+        return {
+          count: indexes.length,
+          data: indexes,
+        };
+      } catch (e) {
+        console.log(
+          `[${this.getKey()}] Failed to processQueue while flushing:`,
+          e
+        );
+        const timestamp = new Date().getTime();
+        await this.redis.hset(this.getKey(`failed:${timestamp}`), {
+          error: e instanceof Error ? e.message : 'Unknown error',
+          data: JSON.stringify(queue.map((item) => item.event)),
+          retries: 0,
+        });
+      }
+    } catch (e) {
+      console.log(`[${this.getKey()}] Failed to getQueue while flushing:`, e);
+    }
+  }
+
+  public async deleteIndexes(indexes: number[]) {
+    const multi = this.redis.multi();
+    indexes.forEach((index) => {
+      multi.lset(this.getKey(), index, DELETE);
+    });
+    multi.lrem(this.getKey(), 0, DELETE);
+    await multi.exec();
+  }
+
+  public async getQueue(limit: number): Promise<QueueItem<T>[]> {
+    const queue = await this.redis.lrange(this.getKey(), 0, limit);
+    return queue
+      .map((item, index) => ({
+        event: this.transformQueueItem(item),
+        index,
+      }))
+      .filter((item): item is QueueItem<T> => item.event !== null);
+  }
+
+  private transformQueueItem(item: string): T | null {
+    try {
+      return JSON.parse(item);
+    } catch (e) {
+      return null;
+    }
+  }
+}

packages/db/src/buffers/event-buffer.ts

@@ -0,0 +1,212 @@
+import { groupBy } from 'ramda';
+import SuperJSON from 'superjson';
+
+import { deepMergeObjects } from '@openpanel/common';
+import { redis, redisPub } from '@openpanel/redis';
+
+import { ch } from '../clickhouse-client';
+import { transformEvent } from '../services/event.service';
+import type {
+  IClickhouseEvent,
+  IServiceCreateEventPayload,
+} from '../services/event.service';
+import type {
+  Find,
+  FindMany,
+  OnCompleted,
+  OnInsert,
+  ProcessQueue,
+  QueueItem,
+} from './buffer';
+import { RedisBuffer } from './buffer';
+
+const sortOldestFirst = (
+  a: QueueItem<IClickhouseEvent>,
+  b: QueueItem<IClickhouseEvent>
+) =>
+  new Date(a.event.created_at).getTime() -
+  new Date(b.event.created_at).getTime();
+
+export class EventBuffer extends RedisBuffer<IClickhouseEvent> {
+  constructor() {
+    super({
+      table: 'events',
+      redis,
+    });
+  }
+
+  public onInsert?: OnInsert<IClickhouseEvent> | undefined = (event) => {
+    redisPub.publish(
+      'event:received',
+      SuperJSON.stringify(transformEvent(event))
+    );
+    this.redis.setex(
+      `live:event:${event.project_id}:${event.profile_id}`,
+      '',
+      60 * 5
+    );
+  };
+
+  public onCompleted?: OnCompleted<IClickhouseEvent> | undefined = (
+    savedEvents
+  ) => {
+    for (const event of savedEvents) {
+      redisPub.publish(
+        'event:saved',
+        SuperJSON.stringify(transformEvent(event))
+      );
+    }
+
+    return savedEvents.map((event) => event.id);
+  };
+
+  public processQueue: ProcessQueue<IClickhouseEvent> = async (queue) => {
+    const itemsToClickhouse = new Set<QueueItem<IClickhouseEvent>>();
+    const itemsToStalled = new Set<QueueItem<IClickhouseEvent>>();
+
+    // Sort data by created_at
+    // oldest first
+    queue.sort(sortOldestFirst);
+
+    // All events thats not a screen_view can be sent to clickhouse
+    // We only need screen_views since we want to calculate the duration of each screen
+    // To do this we need a minimum of 2 screen_views
+    queue
+      .filter(
+        (item) =>
+          item.event.name !== 'screen_view' || item.event.device === 'server'
+      )
+      .forEach((item) => {
+        // Find the last event with data and merge it with the current event
+        // We use profile_id here since this property can be set from backend as well
+        const lastEventWithData = queue
+          .slice(0, item.index)
+          .findLast((lastEvent) => {
+            return (
+              lastEvent.event.project_id === item.event.project_id &&
+              lastEvent.event.profile_id === item.event.profile_id &&
+              lastEvent.event.path !== ''
+            );
+          });
+
+        const event = deepMergeObjects<IClickhouseEvent>(
+          lastEventWithData?.event || {},
+          item.event
+        );
+
+        if (lastEventWithData) {
+          event.properties.__properties_from = lastEventWithData.event.id;
+        }
+
+        return itemsToClickhouse.add({
+          ...item,
+          event,
+        });
+      });
+
+    // Group screen_view events by session_id
+    const grouped = groupBy(
+      (item) => item.event.session_id,
+      queue.filter(
+        (item) =>
+          item.event.name === 'screen_view' && item.event.device !== 'server'
+      )
+    );
+
+    // Iterate over each group
+    for (const [sessionId, screenViews] of Object.entries(grouped)) {
+      if (sessionId === '' || !sessionId) {
+        continue;
+      }
+
+      // If there is only one screen_view event we can send it back to redis since we can't calculate the duration
+      const hasSessionEnd = queue.find(
+        (item) =>
+          item.event.name === 'session_end' &&
+          item.event.session_id === sessionId
+      );
+
+      screenViews
+        ?.slice()
+        .sort(sortOldestFirst)
+        .forEach((item, index) => {
+          const nextScreenView = screenViews[index + 1];
+          // if nextScreenView does not exists we can't calculate the duration (last event in session)
+          if (nextScreenView) {
+            const duration =
+              new Date(nextScreenView.event.created_at).getTime() -
+              new Date(item.event.created_at).getTime();
+            const event = {
+              ...item.event,
+              duration,
+            };
+            event.properties.__duration_from = nextScreenView.event.id;
+            itemsToClickhouse.add({
+              ...item,
+              event,
+            });
+            // push last event in session if we have a session_end event
+          } else if (hasSessionEnd) {
+            itemsToClickhouse.add(item);
+          }
+        });
+    } // for of end
+
+    // Check if we have any events that has been in the queue for more than 24 hour
+    // This should not theoretically happen but if it does we should move them to stalled
+    queue.forEach((item) => {
+      if (
+        !itemsToClickhouse.has(item) &&
+        new Date(item.event.created_at).getTime() <
+          new Date().getTime() - 1000 * 60 * 60 * 24
+      ) {
+        itemsToStalled.add(item);
+      }
+    });
+
+    if (itemsToStalled.size > 0) {
+      const multi = this.redis.multi();
+      for (const item of itemsToStalled) {
+        multi.rpush(this.getKey('stalled'), JSON.stringify(item.event));
+      }
+      await multi.exec();
+    }
+
+    await ch.insert({
+      table: 'events',
+      values: Array.from(itemsToClickhouse).map((item) => item.event),
+      format: 'JSONEachRow',
+    });
+
+    return [
+      ...Array.from(itemsToClickhouse).map((item) => item.index),
+      ...Array.from(itemsToStalled).map((item) => item.index),
+    ];
+  };
+
+  public findMany: FindMany<IClickhouseEvent, IServiceCreateEventPayload> =
+    async (callback) => {
+      return this.getQueue(-1)
+        .then((queue) => {
+          return queue
+            .filter(callback)
+            .map((item) => transformEvent(item.event));
+        })
+        .catch(() => {
+          return [];
+        });
+    };
+
+  public find: Find<IClickhouseEvent, IServiceCreateEventPayload> = async (
+    callback
+  ) => {
+    return this.getQueue(-1)
+      .then((queue) => {
+        const match = queue.find(callback);
+        return match ? transformEvent(match.event) : null;
+      })
+      .catch(() => {
+        return null;
+      });
+  };
+}

packages/db/src/buffers/index.ts

@@ -0,0 +1,5 @@
+import { EventBuffer } from './event-buffer';
+import { ProfileBuffer } from './profile-buffer';
+
+export const eventBuffer = new EventBuffer();
+export const profileBuffer = new ProfileBuffer();

packages/db/src/buffers/profile-buffer.ts

@@ -0,0 +1,114 @@
+import { mergeDeepRight } from 'ramda';
+
+import { toDots } from '@openpanel/common';
+import { redis } from '@openpanel/redis';
+
+import { ch, chQuery } from '../clickhouse-client';
+import type {
+  IClickhouseProfile,
+  IServiceProfile,
+} from '../services/profile.service';
+import { transformProfile } from '../services/profile.service';
+import type {
+  Find,
+  FindMany,
+  OnCompleted,
+  OnInsert,
+  ProcessQueue,
+  QueueItem,
+} from './buffer';
+import { RedisBuffer } from './buffer';
+
+export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {
+  constructor() {
+    super({
+      redis,
+      table: 'profiles',
+      batchSize: 100,
+    });
+  }
+
+  public onInsert?: OnInsert<IClickhouseProfile> | undefined;
+  public onCompleted?: OnCompleted<IClickhouseProfile> | undefined;
+
+  public processQueue: ProcessQueue<IClickhouseProfile> = async (queue) => {
+    const itemsToClickhouse = new Map<string, QueueItem<IClickhouseProfile>>();
+
+    // Combine all writes to the same profile
+    queue.forEach((item) => {
+      const key = item.event.project_id + item.event.id;
+      const existing = itemsToClickhouse.get(key);
+      itemsToClickhouse.set(
+        item.event.project_id + item.event.id,
+        mergeDeepRight(existing ?? {}, item)
+      );
+    });
+
+    const cleanedQueue = Array.from(itemsToClickhouse.values());
+
+    const profiles = await chQuery<IClickhouseProfile>(
+      `SELECT 
+          *
+        FROM profiles
+        WHERE 
+            (id, project_id) IN (${cleanedQueue.map((item) => `('${item.event.id}', '${item.event.project_id}')`).join(',')})
+        ORDER BY
+            created_at DESC`
+    );
+
+    await ch.insert({
+      table: 'profiles',
+      values: cleanedQueue.map((item) => {
+        const profile = profiles.find(
+          (p) =>
+            p.id === item.event.id && p.project_id === item.event.project_id
+        );
+
+        return {
+          id: item.event.id,
+          first_name: item.event.first_name ?? profile?.first_name ?? '',
+          last_name: item.event.last_name ?? profile?.last_name ?? '',
+          email: item.event.email ?? profile?.email ?? '',
+          avatar: item.event.avatar ?? profile?.avatar ?? '',
+          properties: toDots({
+            ...(profile?.properties ?? {}),
+            ...(item.event.properties ?? {}),
+          }),
+          project_id: item.event.project_id ?? profile?.project_id ?? '',
+          created_at: new Date(),
+          is_external: item.event.is_external,
+        };
+      }),
+      clickhouse_settings: {
+        date_time_input_format: 'best_effort',
+      },
+      format: 'JSONEachRow',
+    });
+    return queue.map((item) => item.index);
+  };
+
+  public findMany: FindMany<IClickhouseProfile, IServiceProfile> = async (
+    callback
+  ) => {
+    return this.getQueue(-1)
+      .then((queue) => {
+        return queue
+          .filter(callback)
+          .map((item) => transformProfile(item.event));
+      })
+      .catch(() => {
+        return [];
+      });
+  };
+
+  public find: Find<IClickhouseProfile, IServiceProfile> = async (callback) => {
+    return this.getQueue(-1)
+      .then((queue) => {
+        const match = queue.find(callback);
+        return match ? transformProfile(match.event) : null;
+      })
+      .catch(() => {
+        return null;
+      });
+  };
+}

packages/db/src/clickhouse-client.ts

@@ -1,7 +1,7 @@
 import type { ResponseJSON } from '@clickhouse/client';
 import { createClient } from '@clickhouse/client';
 
-export const ch = createClient({
+export const originalCh = createClient({
   url: process.env.CLICKHOUSE_URL,
   username: process.env.CLICKHOUSE_USER,
   password: process.env.CLICKHOUSE_PASSWORD,
@@ -9,6 +9,53 @@ export const ch = createClient({
   max_open_connections: 10,
   keep_alive: {
     enabled: true,
+    idle_socket_ttl: 5000,
+  },
+  compression: {
+    request: true,
+  },
+});
+
+export const ch = new Proxy(originalCh, {
+  get(target, property, receiver) {
+    if (property === 'insert' || property === 'query') {
+      return async (...args: any[]) => {
+        try {
+          // First attempt
+          if (property in target) {
+            // @ts-expect-error
+            return await target[property](...args);
+          }
+        } catch (error: unknown) {
+          if (
+            error instanceof Error &&
+            error.message.includes('socket hang up')
+          ) {
+            console.error(
+              `Caught socket hang up error on ${property.toString()}, retrying once.`
+            );
+            await new Promise((resolve) => setTimeout(resolve, 500));
+            try {
+              // Retry once
+              if (property in target) {
+                // @ts-expect-error
+                return await target[property](...args);
+              }
+            } catch (retryError) {
+              console.error(
+                `Retry failed for ${property.toString()}:`,
+                retryError
+              );
+              throw retryError; // Rethrow or handle as needed
+            }
+          } else {
+            // Handle other errors or rethrow them
+            throw error;
+          }
+        }
+      };
+    }
+    return Reflect.get(target, property, receiver);
   },
 });
 

packages/db/src/services/event.service.ts

@@ -1,12 +1,12 @@
 import { omit, uniq } from 'ramda';
 import { escape } from 'sqlstring';
-import superjson from 'superjson';
 import { v4 as uuid } from 'uuid';
 
 import { toDots } from '@openpanel/common';
-import { redis, redisPub } from '@openpanel/redis';
+import { redis } from '@openpanel/redis';
 import type { IChartEventFilter } from '@openpanel/validation';
 
+import { eventBuffer } from '../buffers';
 import {
   ch,
   chQuery,
@@ -17,7 +17,7 @@ import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
 import { createSqlBuilder } from '../sql-builder';
 import { getEventFiltersWhereClause } from './chart.service';
-import { getProfileById, getProfiles, upsertProfile } from './profile.service';
+import { getProfiles, upsertProfile } from './profile.service';
 import type { IServiceProfile } from './profile.service';
 
 export interface IClickhouseEvent {
@@ -226,17 +226,14 @@ export async function createEvent(
     payload.profileId = payload.deviceId;
   }
   console.log(
-    `create event ${payload.name} for deviceId: ${payload.deviceId} profileId ${payload.profileId}`
+    `create event ${payload.name} for [deviceId]: ${payload.deviceId} [profileId]: ${payload.profileId} [projectId]: ${payload.projectId} [path]: ${payload.path}`
   );
 
-  const exists = await getProfileById(payload.profileId, payload.projectId);
-  if (!exists && payload.profileId !== '') {
+  if (payload.profileId !== '') {
     await upsertProfile({
       id: String(payload.profileId),
-      isExternal: false,
+      isExternal: payload.profileId !== payload.deviceId,
       projectId: payload.projectId,
-      firstName: '',
-      lastName: '',
       properties: {
         path: payload.path,
         country: payload.country,
@@ -287,25 +284,9 @@ export async function createEvent(
     referrer_type: payload.referrerType ?? '',
   };
 
-  const res = await ch.insert({
-    table: 'events',
-    values: [event],
-    format: 'JSONEachRow',
-    clickhouse_settings: {
-      date_time_input_format: 'best_effort',
-    },
-  });
-
-  redisPub.publish('event', superjson.stringify(transformEvent(event)));
-  redis.set(
-    `live:event:${event.project_id}:${event.profile_id}`,
-    '',
-    'EX',
-    60 * 5
-  );
+  await eventBuffer.insert(event);
 
   return {
-    ...res,
     document: event,
   };
 }
@@ -449,3 +430,27 @@ export function getConversionEventNames(projectId: string) {
     },
   });
 }
+
+export async function getLastScreenViewFromProfileId({
+  profileId,
+  projectId,
+}: {
+  profileId: string;
+  projectId: string;
+}) {
+  const eventInBuffer = await eventBuffer.find(
+    (item) => item.event.profile_id === profileId
+  );
+
+  if (eventInBuffer) {
+    return eventInBuffer;
+  }
+
+  const [eventInDb] = profileId
+    ? await getEvents(
+        `SELECT * FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} AND created_at >= now() - INTERVAL 30 MINUTE ORDER BY created_at DESC LIMIT 1`
+      )
+    : [];
+
+  return eventInDb || null;
+}

packages/db/src/services/profile.service.ts

@@ -1,9 +1,10 @@
 import { escape } from 'sqlstring';
 
-import { toDots, toObject } from '@openpanel/common';
+import { toObject } from '@openpanel/common';
 import type { IChartEventFilter } from '@openpanel/validation';
 
-import { ch, chQuery } from '../clickhouse-client';
+import { profileBuffer } from '../buffers';
+import { chQuery, formatClickhouseDate } from '../clickhouse-client';
 import { createSqlBuilder } from '../sql-builder';
 
 export type IProfileMetrics = {
@@ -66,7 +67,10 @@ export async function getProfiles(ids: string[]) {
   const data = await chQuery<IClickhouseProfile>(
     `SELECT *
     FROM profiles FINAL 
-    WHERE id IN (${ids.map((id) => escape(id)).join(',')})
+    WHERE id IN (${ids
+      .map((id) => escape(id))
+      .filter(Boolean)
+      .join(',')})
     `
   );
 
@@ -172,31 +176,15 @@ export async function upsertProfile({
   projectId,
   isExternal,
 }: IServiceUpsertProfile) {
-  const [profile] = await chQuery<IClickhouseProfile>(
-    `SELECT * FROM profiles WHERE id = ${escape(id)} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`
-  );
-
-  await ch.insert({
-    table: 'profiles',
-    format: 'JSONEachRow',
-    clickhouse_settings: {
-      date_time_input_format: 'best_effort',
-    },
-    values: [
-      {
-        id,
-        first_name: firstName ?? profile?.first_name ?? '',
-        last_name: lastName ?? profile?.last_name ?? '',
-        email: email ?? profile?.email ?? '',
-        avatar: avatar ?? profile?.avatar ?? '',
-        properties: toDots({
-          ...(profile?.properties ?? {}),
-          ...(properties ?? {}),
-        }),
-        project_id: projectId ?? profile?.project_id ?? '',
-        created_at: new Date(),
-        is_external: isExternal,
-      },
-    ],
+  return profileBuffer.insert({
+    id,
+    first_name: firstName!,
+    last_name: lastName!,
+    email: email!,
+    avatar: avatar!,
+    properties: properties as Record<string, string | undefined>,
+    project_id: projectId,
+    created_at: formatClickhouseDate(new Date()),
+    is_external: isExternal,
   });
 }

packages/logger/index.ts

@@ -3,7 +3,7 @@ import pino from 'pino';
 
 export function createLogger({ dataset }: { dataset: string }) {
   const targets: TransportTargetOptions[] =
-    process.env.NODE_ENV === 'production'
+    process.env.NODE_ENV === 'production' && process.env.BASELIME_API_KEY
       ? [
           {
             target: '@baselime/pino-transport',

packages/queue/package.json

@@ -9,7 +9,7 @@
   },
   "dependencies": {
     "@openpanel/db": "workspace:*",
-    "bullmq": "^5.1.1"
+    "bullmq": "^5.8.7"
   },
   "devDependencies": {
     "@openpanel/sdk": "workspace:*",
@@ -28,4 +28,4 @@
     ]
   },
   "prettier": "@openpanel/prettier-config"
-}
+}

packages/queue/src/queues.ts

@@ -22,8 +22,7 @@ export interface EventsQueuePayloadIncomingEvent {
     };
     currentDeviceId: string;
     previousDeviceId: string;
-    currentDeviceIdDeprecated: string;
-    previousDeviceIdDeprecated: string;
+    priority: boolean;
   };
 }
 export interface EventsQueuePayloadCreateEvent {
@@ -32,17 +31,36 @@ export interface EventsQueuePayloadCreateEvent {
 }
 export interface EventsQueuePayloadCreateSessionEnd {
   type: 'createSessionEnd';
-  payload: Pick<IServiceCreateEventPayload, 'deviceId'>;
+  payload: Pick<
+    IServiceCreateEventPayload,
+    'deviceId' | 'sessionId' | 'profileId'
+  >;
 }
+
+// TODO: Rename `EventsQueuePayloadCreateSessionEnd`
+export type SessionsQueuePayload = EventsQueuePayloadCreateSessionEnd;
+
 export type EventsQueuePayload =
   | EventsQueuePayloadCreateEvent
   | EventsQueuePayloadCreateSessionEnd
   | EventsQueuePayloadIncomingEvent;
 
-export interface CronQueuePayload {
+export type CronQueuePayloadSalt = {
   type: 'salt';
   payload: undefined;
-}
+};
+export type CronQueuePayloadFlushEvents = {
+  type: 'flushEvents';
+  payload: undefined;
+};
+export type CronQueuePayloadFlushProfiles = {
+  type: 'flushProfiles';
+  payload: undefined;
+};
+export type CronQueuePayload =
+  | CronQueuePayloadSalt
+  | CronQueuePayloadFlushEvents
+  | CronQueuePayloadFlushProfiles;
 
 export const eventsQueue = new Queue<EventsQueuePayload>('events', {
   connection,
@@ -51,6 +69,13 @@ export const eventsQueue = new Queue<EventsQueuePayload>('events', {
   },
 });
 
+export const sessionsQueue = new Queue<SessionsQueuePayload>('sessions', {
+  connection,
+  defaultJobOptions: {
+    removeOnComplete: 10,
+  },
+});
+
 export const cronQueue = new Queue<CronQueuePayload>('cron', {
   connection,
   defaultJobOptions: {

packages/queue/src/utils.ts

@@ -24,7 +24,7 @@ export async function findJobByPrefix<T>(
   async function getJob(index: number) {
     if (index >= filtered.length) return null;
 
-    const key = filtered[index]?.replace(/^bull:events:/, '');
+    const key = filtered[index]?.replace(/^bull:(\w+):/, '');
     // return new Promise((resolve) => )
     if (key) {
       const job = await queue.getJob(key);

packages/redis/cachable.ts

@@ -4,14 +4,25 @@ export function cacheable<T extends (...args: any) => any>(
   fn: T,
   expire: number
 ) {
-  return async function (...args: Parameters<T>): Promise<ReturnType<T>> {
+  return async function (
+    ...args: Parameters<T>
+  ): Promise<Awaited<ReturnType<T>>> {
+    // JSON.stringify here is not bullet proof since ordering of object keys matters etc
     const key = `cachable:${fn.name}:${JSON.stringify(args)}`;
     const cached = await redis.get(key);
     if (cached) {
-      return JSON.parse(cached);
+      try {
+        return JSON.parse(cached);
+      } catch (e) {
+        console.error('Failed to parse cache', e);
+      }
     }
     const result = await fn(...(args as any));
-    redis.setex(key, expire, JSON.stringify(result));
+
+    if (result !== undefined || result !== null) {
+      redis.setex(key, expire, JSON.stringify(result));
+    }
+
     return result;
   };
 }

packages/redis/package.json

@@ -9,7 +9,7 @@
     "with-env": "dotenv -e ../../.env -c --"
   },
   "dependencies": {
-    "ioredis": "^5.3.2"
+    "ioredis": "^5.4.1"
   },
   "devDependencies": {
     "@openpanel/eslint-config": "workspace:*",
@@ -28,4 +28,4 @@
     ]
   },
   "prettier": "@openpanel/prettier-config"
-}
+}

packages/redis/redis.ts

@@ -2,9 +2,12 @@ import type { RedisOptions } from 'ioredis';
 import Redis from 'ioredis';
 
 const options: RedisOptions = {
-  connectTimeout: 10000,
+  connectTimeout: 30000,
+  maxRetriesPerRequest: null,
 };
 
+export { Redis };
+
 export const redis = new Redis(process.env.REDIS_URL!, options);
 export const redisSub = new Redis(process.env.REDIS_URL!, options);
 export const redisPub = new Redis(process.env.REDIS_URL!, options);