prep events partition

apps/api/scripts/migrate-origins.ts

@@ -1,8 +1,8 @@
-import { ch, chQuery } from '@openpanel/db';
+import { ch, chQuery, TABLE_NAMES } from '@openpanel/db';
 
 async function main() {
   const projects = await chQuery(
-    `SELECT distinct project_id FROM events ORDER BY project_id`
+    `SELECT distinct project_id FROM ${TABLE_NAMES.events} ORDER BY project_id`
   );
   const withOrigin = [];
 
@@ -10,10 +10,10 @@ async function main() {
     try {
       const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([
         await chQuery(
-          `SELECT * FROM events WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`
+          `SELECT * FROM ${TABLE_NAMES.events} WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`
         ),
         await chQuery(
-          `SELECT * FROM events WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`
+          `SELECT * FROM ${TABLE_NAMES.events} WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`
         ),
       ]);
 
@@ -22,7 +22,7 @@ async function main() {
         console.log(`- Origin: ${eventWithOrigin[0].origin}`);
         withOrigin.push(project.project_id);
         const events = await chQuery(
-          `SELECT count(*) as count FROM events WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`
+          `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`
         );
         console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);
         await ch.command({

apps/api/src/controllers/live.controller.ts

@@ -10,6 +10,7 @@ import {
   getEvents,
   getLiveVisitors,
   getProfileById,
+  TABLE_NAMES,
   transformMinimalEvent,
 } from '@openpanel/db';
 import { redis, redisPub, redisSub } from '@openpanel/redis';
@@ -28,8 +29,7 @@ export async function testVisitors(
   reply: FastifyReply
 ) {
   const events = await getEvents(
-    `SELECT * FROM events LIMIT 500`
+    `SELECT * FROM ${TABLE_NAMES.events} LIMIT 500`
-    // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
   );
   const event = events[Math.floor(Math.random() * events.length)];
   if (!event) {
@@ -55,8 +55,7 @@ export async function testEvents(
   reply: FastifyReply
 ) {
   const events = await getEvents(
-    `SELECT * FROM events LIMIT 500`
+    `SELECT * FROM ${TABLE_NAMES.events} LIMIT 500`
-    // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
   );
   const event = events[Math.floor(Math.random() * events.length)];
   if (!event) {

apps/api/src/index.ts

@@ -7,7 +7,7 @@ import Fastify from 'fastify';
 import metricsPlugin from 'fastify-metrics';
 
 import { round } from '@openpanel/common';
-import { chQuery, db } from '@openpanel/db';
+import { chQuery, db, TABLE_NAMES } from '@openpanel/db';
 import type { IServiceClient } from '@openpanel/db';
 import { eventsQueue } from '@openpanel/queue';
 import { redis, redisPub } from '@openpanel/redis';
@@ -101,7 +101,9 @@ const startServer = async () => {
       const redisRes = await withTimings(redis.keys('*'));
       const dbRes = await withTimings(db.project.findFirst());
       const queueRes = await withTimings(eventsQueue.getCompleted());
-      const chRes = await withTimings(chQuery('SELECT * FROM events LIMIT 1'));
+      const chRes = await withTimings(
+        chQuery(`SELECT * FROM ${TABLE_NAMES.events} LIMIT 1`)
+      );
       const status = redisRes && dbRes && queueRes && chRes ? 200 : 500;
 
       reply.status(status).send({

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx

@@ -1,7 +1,7 @@
 import withLoadingWidget from '@/hocs/with-loading-widget';
 import { escape } from 'sqlstring';
 
-import { db, getEvents } from '@openpanel/db';
+import { db, getEvents, TABLE_NAMES } from '@openpanel/db';
 
 import { EventConversionsList } from './event-conversions-list';
 
@@ -22,7 +22,7 @@ async function EventConversionsListServer({ projectId }: Props) {
   }
 
   const events = await getEvents(
-    `SELECT * FROM events WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,
+    `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,
     {
       profile: true,
       meta: true,

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx

@@ -1,7 +1,7 @@
 import withLoadingWidget from '@/hocs/with-loading-widget';
 import { escape } from 'sqlstring';
 
-import { chQuery } from '@openpanel/db';
+import { chQuery, TABLE_NAMES } from '@openpanel/db';
 
 import MostEvents from './most-events';
 
@@ -12,7 +12,7 @@ type Props = {
 
 const MostEventsServer = async ({ projectId, profileId }: Props) => {
   const data = await chQuery<{ count: number; name: string }>(
-    `SELECT count(*) as count, name FROM events WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC`
+    `SELECT count(*) as count, name FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC`
   );
   return <MostEvents data={data} />;
 };

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx

@@ -1,7 +1,7 @@
 import withLoadingWidget from '@/hocs/with-loading-widget';
 import { escape } from 'sqlstring';
 
-import { chQuery } from '@openpanel/db';
+import { chQuery, TABLE_NAMES } from '@openpanel/db';
 
 import PopularRoutes from './popular-routes';
 
@@ -12,7 +12,7 @@ type Props = {
 
 const PopularRoutesServer = async ({ projectId, profileId }: Props) => {
   const data = await chQuery<{ count: number; path: string }>(
-    `SELECT count(*) as count, path FROM events WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC`
+    `SELECT count(*) as count, path FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC`
   );
   return <PopularRoutes data={data} />;
 };

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx

@@ -1,7 +1,7 @@
 import withLoadingWidget from '@/hocs/with-loading-widget';
 import { escape } from 'sqlstring';
 
-import { chQuery } from '@openpanel/db';
+import { chQuery, TABLE_NAMES } from '@openpanel/db';
 
 import ProfileActivity from './profile-activity';
 
@@ -12,7 +12,7 @@ type Props = {
 
 const ProfileActivityServer = async ({ projectId, profileId }: Props) => {
   const data = await chQuery<{ count: number; date: string }>(
-    `SELECT count(*) as count, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC`
+    `SELECT count(*) as count, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC`
   );
   return <ProfileActivity data={data} />;
 };

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx

@@ -7,7 +7,7 @@ import { Widget, WidgetBody, WidgetHead } from '@/components/widget';
 import { cn } from '@/utils/cn';
 import { escape } from 'sqlstring';
 
-import { chQuery } from '@openpanel/db';
+import { chQuery, TABLE_NAMES } from '@openpanel/db';
 
 interface Props {
   projectId: string;
@@ -21,7 +21,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) {
   // Days since last event from users
   // group by days
   const res = await chQuery<Row>(
-    `SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM events where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51`
+    `SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM ${TABLE_NAMES.events} where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51`
   );
 
   const maxValue = Math.max(...res.map((x) => x.count));
@@ -37,7 +37,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) {
       <Tooltip>
         <TooltipTrigger asChild>
           <div
-            className={cn('bg-highlight aspect-square w-full shrink-0 rounded')}
+            className={cn('aspect-square w-full shrink-0 rounded bg-highlight')}
             style={{
               opacity: calculateRatio(item.count),
             }}

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-top/index.tsx

@@ -7,7 +7,7 @@ import { getProfileName } from '@/utils/getters';
 import Link from 'next/link';
 import { escape } from 'sqlstring';
 
-import { chQuery, getProfiles } from '@openpanel/db';
+import { chQuery, getProfiles, TABLE_NAMES } from '@openpanel/db';
 
 interface Props {
   projectId: string;
@@ -18,7 +18,7 @@ async function ProfileTopServer({ organizationSlug, projectId }: Props) {
   // Days since last event from users
   // group by days
   const res = await chQuery<{ profile_id: string; count: number }>(
-    `SELECT profile_id, count(*) as count from events where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50`
+    `SELECT profile_id, count(*) as count from ${TABLE_NAMES.events} where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50`
   );
   const profiles = await getProfiles(res.map((r) => r.profile_id));
   const list = res.map((item) => {

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx

@@ -1,7 +1,7 @@
 import { subMinutes } from 'date-fns';
 import { escape } from 'sqlstring';
 
-import { chQuery, formatClickhouseDate } from '@openpanel/db';
+import { chQuery, formatClickhouseDate, TABLE_NAMES } from '@openpanel/db';
 
 import type { Coordinate } from './coordinates';
 import Map from './map';
@@ -11,7 +11,7 @@ type Props = {
 };
 const RealtimeMap = async ({ projectId }: Props) => {
   const res = await chQuery<Coordinate>(
-    `SELECT DISTINCT city, longitude as long, latitude as lat FROM events WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
+    `SELECT DISTINCT city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
   );
 
   return <Map markers={res} />;

apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx

@@ -1,6 +1,6 @@
 import { escape } from 'sqlstring';
 
-import { getEvents } from '@openpanel/db';
+import { getEvents, TABLE_NAMES } from '@openpanel/db';
 
 import LiveEvents from './live-events';
 
@@ -10,7 +10,7 @@ type Props = {
 };
 const RealtimeLiveEventsServer = async ({ projectId, limit = 30 }: Props) => {
   const events = await getEvents(
-    `SELECT * FROM events WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`,
+    `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`,
     {
       profile: true,
     }

apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx

@@ -5,6 +5,7 @@ import {
   getCurrentOrganizations,
   getEvents,
   getProjectWithClients,
+  TABLE_NAMES,
 } from '@openpanel/db';
 
 import OnboardingVerify from './onboarding-verify';
@@ -24,7 +25,7 @@ const Verify = async ({ params: { projectId } }: Props) => {
   const [project, events] = await Promise.all([
     await getProjectWithClients(projectId),
     getEvents(
-      `SELECT * FROM events WHERE project_id = ${escape(projectId)} LIMIT 100`
+      `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} LIMIT 100`
     ),
   ]);
 

apps/dashboard/src/components/projects/project-card.tsx

@@ -3,7 +3,7 @@ import { shortNumber } from '@/hooks/useNumerFormatter';
 import { escape } from 'sqlstring';
 
 import type { IServiceProject } from '@openpanel/db';
-import { chQuery } from '@openpanel/db';
+import { chQuery, TABLE_NAMES } from '@openpanel/db';
 
 import { ChartSSR } from '../chart-ssr';
 import { FadeIn } from '../fade-in';
@@ -34,7 +34,7 @@ function ProjectCard({ id, name, organizationSlug }: IServiceProject) {
 
 async function ProjectChart({ id }: { id: string }) {
   const chart = await chQuery<{ value: number; date: string }>(
-    `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`
+    `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`
   );
 
   return (
@@ -62,13 +62,13 @@ async function ProjectMetrics({ id }: { id: string }) {
     `
       SELECT
       (
-        SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)}
+        SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)}
       ) as total, 
       (
-        SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month'
+        SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month'
       ) as month,
       (
-        SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day'
+        SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day'
       ) as day
     `
   );

apps/public/src/app/hero.tsx

@@ -1,6 +1,6 @@
 import { ALink } from '@/components/ui/button';
 
-import { chQuery } from '@openpanel/db';
+import { chQuery, TABLE_NAMES } from '@openpanel/db';
 
 import AnimatedText from './animated-text';
 import { Heading1, Lead2 } from './copy';
@@ -15,7 +15,7 @@ function shortNumber(num: number) {
 
 export async function Hero() {
   const projects = await chQuery<{ project_id: string; count: number }>(
-    'SELECT project_id, count(*) as count from events GROUP by project_id order by count()'
+    `SELECT project_id, count(*) as count from ${TABLE_NAMES.events} GROUP by project_id order by count()`
   );
   const projectCount = projects.length;
   const eventCount = projects.reduce((acc, { count }) => acc + count, 0);

apps/worker/scripts/debug.ts

@@ -1,7 +1,7 @@
 import { escape } from 'sqlstring';
 
 import type { IClickhouseEvent } from '@openpanel/db';
-import { chQuery, eventBuffer } from '@openpanel/db';
+import { chQuery, eventBuffer, TABLE_NAMES } from '@openpanel/db';
 import { sessionsQueue } from '@openpanel/queue/src/queues';
 import { redis } from '@openpanel/redis';
 
@@ -70,7 +70,7 @@ async function debugStalledEvents() {
 
   if (stalledEvents.length > 0) {
     const res = await chQuery(
-      `SELECT * FROM events WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`
+      `SELECT * FROM ${TABLE_NAMES.events} WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`
     );
 
     stalledEvents.forEach((item) => {

apps/worker/src/jobs/events.create-session-end.ts

@@ -1,7 +1,12 @@
 import type { Job } from 'bullmq';
 
 import { getTime } from '@openpanel/common';
-import { createEvent, eventBuffer, getEvents } from '@openpanel/db';
+import {
+  createEvent,
+  eventBuffer,
+  getEvents,
+  TABLE_NAMES,
+} from '@openpanel/db';
 import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';
 
 export async function createSessionEnd(
@@ -13,12 +18,12 @@ export async function createSessionEnd(
   );
 
   const sql = `
-  SELECT * FROM events 
+  SELECT * FROM ${TABLE_NAMES.events} 
   WHERE 
     session_id = '${payload.sessionId}' 
     AND created_at >= (
       SELECT created_at 
-      FROM events
+      FROM ${TABLE_NAMES.events}
       WHERE 
         session_id = '${payload.sessionId}' 
         AND name = 'session_start'

apps/worker/src/jobs/events.ts

@@ -1,12 +1,13 @@
 import type { Job } from 'bullmq';
 import { escape } from 'sqlstring';
 
-import { chQuery, db } from '@openpanel/db';
+import { chQuery, db, TABLE_NAMES } from '@openpanel/db';
 import type {
   EventsQueuePayload,
   EventsQueuePayloadCreateSessionEnd,
   EventsQueuePayloadIncomingEvent,
 } from '@openpanel/queue';
+import { redis } from '@openpanel/redis';
 
 import { createSessionEnd } from './events.create-session-end';
 import { incomingEvent } from './events.incoming-event';
@@ -26,7 +27,7 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {
 
 async function updateEventsCount(projectId: string) {
   const res = await chQuery<{ count: number }>(
-    `SELECT count(*) as count FROM events WHERE project_id = ${escape(projectId)}`
+    `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`
   );
   const count = res[0]?.count;
   if (count) {

packages/db/clickhouse_init.sql

@@ -34,6 +34,51 @@ CREATE TABLE IF NOT EXISTS openpanel.events (
 ORDER BY
   (project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
 
+CREATE TABLE IF NOT EXISTS openpanel.events_v2 (
+  `id` UUID DEFAULT generateUUIDv4(),
+  `name` String,
+  `device_id` String,
+  `profile_id` String,
+  `project_id` String,
+  `session_id` String,
+  `path` String,
+  `origin` String,
+  `referrer` String,
+  `referrer_name` String,
+  `referrer_type` String,
+  `duration` UInt64,
+  `properties` Map(String, String),
+  `created_at` DateTime64(3),
+  `country` String,
+  `city` String,
+  `region` String,
+  `longitude` Nullable(Float32),
+  `latitude` Nullable(Float32),
+  `os` String,
+  `os_version` String,
+  `browser` String,
+  `browser_version` String,
+  -- device: mobile/desktop/tablet
+  `device` String,
+  -- brand: (Samsung, OnePlus)
+  `brand` String,
+  -- model: (Samsung Galaxy, iPhone X)
+  `model` String
+) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at)
+ORDER BY
+  (project_id, created_at, profile_id) SETTINGS index_granularity = 8192;
+
+ALTER TABLE
+  events DROP COLUMN utm_source,
+  DROP COLUMN utm_medium,
+  DROP COLUMN utm_campaign,
+  DROP COLUMN utm_term,
+  DROP COLUMN utm_content,
+  DROP COLUMN sdk,
+  DROP COLUMN sdk_version,
+  DROP COLUMN client_type,
+  DROP COLUMN continent;
+
 CREATE TABLE IF NOT EXISTS openpanel.events_bots (
   `id` UUID DEFAULT generateUUIDv4(),
   `project_id` String,

packages/db/src/buffers/event-buffer.ts

@@ -4,7 +4,7 @@ import SuperJSON from 'superjson';
 import { deepMergeObjects } from '@openpanel/common';
 import { redis, redisPub } from '@openpanel/redis';
 
-import { ch } from '../clickhouse-client';
+import { ch, TABLE_NAMES } from '../clickhouse-client';
 import { transformEvent } from '../services/event.service';
 import type {
   IClickhouseEvent,
@@ -30,7 +30,7 @@ const sortOldestFirst = (
 export class EventBuffer extends RedisBuffer<IClickhouseEvent> {
   constructor() {
     super({
-      table: 'events',
+      table: TABLE_NAMES.events,
       redis,
     });
   }
@@ -176,7 +176,7 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {
     }
 
     await ch.insert({
-      table: 'events',
+      table: TABLE_NAMES.events,
       values: Array.from(itemsToClickhouse).map((item) => item.event),
       format: 'JSONEachRow',
     });

packages/db/src/clickhouse-client.ts

@@ -1,6 +1,11 @@
 import type { ResponseJSON } from '@clickhouse/client';
 import { createClient } from '@clickhouse/client';
 
+export const TABLE_NAMES = {
+  events: 'events',
+  profiles: 'profiles',
+};
+
 export const originalCh = createClient({
   url: process.env.CLICKHOUSE_URL,
   username: process.env.CLICKHOUSE_USER,

packages/db/src/services/chart.service.ts

@@ -6,7 +6,7 @@ import type {
   IGetChartDataInput,
 } from '@openpanel/validation';
 
-import { formatClickhouseDate } from '../clickhouse-client';
+import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client';
 import { createSqlBuilder } from '../sql-builder';
 
 export function getChartSql({
@@ -94,7 +94,7 @@ export function getChartSql({
 
   if (event.segment === 'one_event_per_user') {
     sb.from = `(
-      SELECT DISTINCT ON (profile_id) * from events WHERE ${join(
+      SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} WHERE ${join(
         sb.where,
         ' AND '
       )}

packages/db/src/services/event.service.ts

@@ -12,6 +12,7 @@ import {
   chQuery,
   convertClickhouseDateToJs,
   formatClickhouseDate,
+  TABLE_NAMES,
 } from '../clickhouse-client';
 import type { EventMeta, Prisma } from '../prisma-client';
 import { db } from '../prisma-client';
@@ -323,7 +324,7 @@ export async function getEventList({
   sb.where.projectId = `project_id = ${escape(projectId)}`;
 
   if (profileId) {
-    sb.where.deviceId = `device_id IN (SELECT device_id as did FROM events WHERE profile_id = ${escape(profileId)} group by did)`;
+    sb.where.deviceId = `device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} group by did)`;
   }
 
   if (startDate && endDate) {
@@ -448,7 +449,7 @@ export async function getLastScreenViewFromProfileId({
 
   const [eventInDb] = profileId
     ? await getEvents(
-        `SELECT * FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} AND created_at >= now() - INTERVAL 30 MINUTE ORDER BY created_at DESC LIMIT 1`
+        `SELECT * FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} AND created_at >= now() - INTERVAL 30 MINUTE ORDER BY created_at DESC LIMIT 1`
       )
     : [];
 

packages/db/src/services/profile.service.ts

@@ -4,7 +4,11 @@ import { toObject } from '@openpanel/common';
 import type { IChartEventFilter } from '@openpanel/validation';
 
 import { profileBuffer } from '../buffers';
-import { chQuery, formatClickhouseDate } from '../clickhouse-client';
+import {
+  chQuery,
+  formatClickhouseDate,
+  TABLE_NAMES,
+} from '../clickhouse-client';
 import { createSqlBuilder } from '../sql-builder';
 
 export type IProfileMetrics = {
@@ -18,19 +22,19 @@ export type IProfileMetrics = {
 export function getProfileMetrics(profileId: string, projectId: string) {
   return chQuery<IProfileMetrics>(`
     WITH lastSeen AS (
-      SELECT max(created_at) as lastSeen FROM events WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
     ),
     firstSeen AS (
-      SELECT min(created_at) as firstSeen FROM events WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
     ),
     screenViews AS (
-      SELECT count(*) as screenViews FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
     ),
     sessions AS (
-      SELECT count(*) as sessions FROM events WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
     ),
     duration AS (
-      SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM events WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
+      SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)}
     )
     SELECT lastSeen, firstSeen, screenViews, sessions, durationAvg, durationP90 FROM lastSeen, firstSeen, screenViews,sessions, duration
   `).then((data) => data[0]!);

packages/db/src/services/retention.service.ts

@@ -1,6 +1,6 @@
 import { escape } from 'sqlstring';
 
-import { chQuery } from '../clickhouse-client';
+import { chQuery, TABLE_NAMES } from '../clickhouse-client';
 
 type IGetWeekRetentionInput = {
   projectId: string;
@@ -15,7 +15,7 @@ WITH
       SELECT
           profile_id,
           max(toWeek(created_at)) AS last_seen
-      FROM events
+      FROM ${TABLE_NAMES.events}
       WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)
       GROUP BY profile_id
   ),
@@ -24,7 +24,7 @@ WITH
       SELECT
           profile_id,
           min(toWeek(created_at)) AS first_seen
-      FROM events
+      FROM ${TABLE_NAMES.events}
       WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id)
       GROUP BY profile_id
   ),
@@ -79,8 +79,8 @@ export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {
       countDistinct(events.profile_id) AS active_users,
       countDistinct(future_events.profile_id) AS retained_users,
       (100 * (countDistinct(future_events.profile_id) / CAST(countDistinct(events.profile_id), 'float'))) AS retention
-    FROM events
+    FROM ${TABLE_NAMES.events} as events
-    LEFT JOIN events AS future_events ON 
+    LEFT JOIN ${TABLE_NAMES.events} AS future_events ON 
       events.profile_id = future_events.profile_id
       AND toStartOfWeek(events.created_at) = toStartOfWeek(future_events.created_at - toIntervalWeek(1))
       AND future_events.profile_id != future_events.device_id
@@ -140,7 +140,7 @@ export function getRetentionLastSeenSeries({
         SELECT
             max(created_at) AS last_active,
             profile_id
-        FROM events
+        FROM ${TABLE_NAMES.events}
         WHERE (project_id = ${escape(projectId)}) AND (device_id != profile_id)
         GROUP BY profile_id
     )

packages/db/src/sql-builder.ts

@@ -1,3 +1,5 @@
+import { TABLE_NAMES } from './clickhouse-client';
+
 export interface SqlBuilderObject {
   where: Record<string, string>;
   having: Record<string, string>;
@@ -15,7 +17,7 @@ export function createSqlBuilder() {
 
   const sb: SqlBuilderObject = {
     where: {},
-    from: 'events',
+    from: TABLE_NAMES.events,
     select: {},
     groupBy: {},
     orderBy: {},

packages/trpc/src/routers/chart.helpers.ts

@@ -34,6 +34,7 @@ import {
   getChartSql,
   getEventFiltersWhereClause,
   getProfiles,
+  TABLE_NAMES,
 } from '@openpanel/db';
 import type {
   FinalChart,
@@ -293,7 +294,7 @@ export async function getFunnelData({
   const innerSql = `SELECT
     session_id,
     windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
-  FROM events
+  FROM ${TABLE_NAMES.events}
   WHERE 
     project_id = ${escape(projectId)} AND 
     created_at >= '${formatClickhouseDate(startDate)}' AND 
@@ -306,7 +307,7 @@ export async function getFunnelData({
   const [funnelRes, sessionRes] = await Promise.all([
     chQuery<{ level: number; count: number }>(sql),
     chQuery<{ count: number }>(
-      `SELECT count(name) as count FROM events WHERE project_id = ${escape(projectId)} AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')`
+      `SELECT count(name) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')`
     ),
   ]);
 
@@ -409,7 +410,7 @@ export async function getFunnelStep({
   const innerSql = `SELECT
     session_id,
     windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level
-  FROM events
+  FROM ${TABLE_NAMES.events}
   WHERE 
     project_id = ${escape(projectId)} AND 
     created_at >= '${formatClickhouseDate(startDate)}' AND 
@@ -421,7 +422,7 @@ export async function getFunnelStep({
     SELECT 
       DISTINCT e.profile_id as id
     FROM sessions s
-    JOIN events e ON s.session_id = e.session_id
+    JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_id
     WHERE 
       s.level = ${step} AND
       e.project_id = ${escape(projectId)} AND 

packages/trpc/src/routers/chart.ts

@@ -3,7 +3,7 @@ import { escape } from 'sqlstring';
 import { z } from 'zod';
 
 import { average, max, min, round, slug, sum } from '@openpanel/common';
-import { chQuery, createSqlBuilder, db } from '@openpanel/db';
+import { chQuery, createSqlBuilder, db, TABLE_NAMES } from '@openpanel/db';
 import { zChartInput } from '@openpanel/validation';
 import type {
   FinalChart,
@@ -17,7 +17,6 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc';
 import {
   getChart,
   getChartPrevStartEndDate,
-  getChartSeries,
   getChartStartEndDate,
   getFunnelData,
   getFunnelStep,
@@ -28,7 +27,7 @@ export const chartRouter = createTRPCRouter({
     .input(z.object({ projectId: z.string() }))
     .query(async ({ input: { projectId } }) => {
       const events = await chQuery<{ name: string }>(
-        `SELECT DISTINCT name FROM events WHERE project_id = ${escape(projectId)}`
+        `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`
       );
 
       return [
@@ -43,7 +42,7 @@ export const chartRouter = createTRPCRouter({
     .input(z.object({ event: z.string().optional(), projectId: z.string() }))
     .query(async ({ input: { projectId, event } }) => {
       const events = await chQuery<{ keys: string[] }>(
-        `SELECT distinct mapKeys(properties) as keys from events where ${
+        `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${
           event && event !== '*' ? `name = ${escape(event)} AND ` : ''
         } project_id = ${escape(projectId)};`
       );