diff --git a/apps/api/scripts/migrate-origins.ts b/apps/api/scripts/migrate-origins.ts index 7f365e85..0c8698bd 100644 --- a/apps/api/scripts/migrate-origins.ts +++ b/apps/api/scripts/migrate-origins.ts @@ -1,8 +1,8 @@ -import { ch, chQuery } from '@openpanel/db'; +import { ch, chQuery, TABLE_NAMES } from '@openpanel/db'; async function main() { const projects = await chQuery( - `SELECT distinct project_id FROM events ORDER BY project_id` + `SELECT distinct project_id FROM ${TABLE_NAMES.events} ORDER BY project_id` ); const withOrigin = []; @@ -10,10 +10,10 @@ async function main() { try { const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([ await chQuery( - `SELECT * FROM events WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1` + `SELECT * FROM ${TABLE_NAMES.events} WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1` ), await chQuery( - `SELECT * FROM events WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1` + `SELECT * FROM ${TABLE_NAMES.events} WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1` ), ]); @@ -22,7 +22,7 @@ async function main() { console.log(`- Origin: ${eventWithOrigin[0].origin}`); withOrigin.push(project.project_id); const events = await chQuery( - `SELECT count(*) as count FROM events WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''` + `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''` ); console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`); await ch.command({ diff --git a/apps/api/src/controllers/live.controller.ts b/apps/api/src/controllers/live.controller.ts index e0e69102..ba783fe2 100644 --- a/apps/api/src/controllers/live.controller.ts +++ b/apps/api/src/controllers/live.controller.ts @@ -10,6 +10,7 @@ import { getEvents, getLiveVisitors, getProfileById, + TABLE_NAMES, transformMinimalEvent, } from '@openpanel/db'; import { redis, redisPub, redisSub } from '@openpanel/redis'; @@ -28,8 +29,7 @@ export async function testVisitors( reply: FastifyReply ) { const events = await getEvents( - `SELECT * FROM events LIMIT 500` - // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500` + `SELECT * FROM ${TABLE_NAMES.events} LIMIT 500` ); const event = events[Math.floor(Math.random() * events.length)]; if (!event) { @@ -55,8 +55,7 @@ export async function testEvents( reply: FastifyReply ) { const events = await getEvents( - `SELECT * FROM events LIMIT 500` - // `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500` + `SELECT * FROM ${TABLE_NAMES.events} LIMIT 500` ); const event = events[Math.floor(Math.random() * events.length)]; if (!event) { diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 9d918d73..57e596bc 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -7,7 +7,7 @@ import Fastify from 'fastify'; import metricsPlugin from 'fastify-metrics'; import { round } from '@openpanel/common'; -import { chQuery, db } from '@openpanel/db'; +import { chQuery, db, TABLE_NAMES } from '@openpanel/db'; import type { IServiceClient } from '@openpanel/db'; import { eventsQueue } from '@openpanel/queue'; import { redis, redisPub } from '@openpanel/redis'; @@ -101,7 +101,9 @@ const startServer = async () => { const redisRes = await withTimings(redis.keys('*')); const dbRes = await withTimings(db.project.findFirst()); const queueRes = await withTimings(eventsQueue.getCompleted()); - const chRes = await withTimings(chQuery('SELECT * FROM events LIMIT 1')); + const chRes = await withTimings( + chQuery(`SELECT * FROM ${TABLE_NAMES.events} LIMIT 1`) + ); const status = redisRes && dbRes && queueRes && chRes ? 200 : 500; reply.status(status).send({ diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx index 338ae0ac..fb204124 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx @@ -1,7 +1,7 @@ import withLoadingWidget from '@/hocs/with-loading-widget'; import { escape } from 'sqlstring'; -import { db, getEvents } from '@openpanel/db'; +import { db, getEvents, TABLE_NAMES } from '@openpanel/db'; import { EventConversionsList } from './event-conversions-list'; @@ -22,7 +22,7 @@ async function EventConversionsListServer({ projectId }: Props) { } const events = await getEvents( - `SELECT * FROM events WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`, + `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`, { profile: true, meta: true, diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx index f52cd411..8be37ccd 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx @@ -1,7 +1,7 @@ import withLoadingWidget from '@/hocs/with-loading-widget'; import { escape } from 'sqlstring'; -import { chQuery } from '@openpanel/db'; +import { chQuery, TABLE_NAMES } from '@openpanel/db'; import MostEvents from './most-events'; @@ -12,7 +12,7 @@ type Props = { const MostEventsServer = async ({ projectId, profileId }: Props) => { const data = await chQuery<{ count: number; name: string }>( - `SELECT count(*) as count, name FROM events WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC` + `SELECT count(*) as count, name FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC` ); return ; }; diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx index 64980052..66ea3fb4 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx @@ -1,7 +1,7 @@ import withLoadingWidget from '@/hocs/with-loading-widget'; import { escape } from 'sqlstring'; -import { chQuery } from '@openpanel/db'; +import { chQuery, TABLE_NAMES } from '@openpanel/db'; import PopularRoutes from './popular-routes'; @@ -12,7 +12,7 @@ type Props = { const PopularRoutesServer = async ({ projectId, profileId }: Props) => { const data = await chQuery<{ count: number; path: string }>( - `SELECT count(*) as count, path FROM events WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC` + `SELECT count(*) as count, path FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC` ); return ; }; diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx index 0bd724a0..37784b7a 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx @@ -1,7 +1,7 @@ import withLoadingWidget from '@/hocs/with-loading-widget'; import { escape } from 'sqlstring'; -import { chQuery } from '@openpanel/db'; +import { chQuery, TABLE_NAMES } from '@openpanel/db'; import ProfileActivity from './profile-activity'; @@ -12,7 +12,7 @@ type Props = { const ProfileActivityServer = async ({ projectId, profileId }: Props) => { const data = await chQuery<{ count: number; date: string }>( - `SELECT count(*) as count, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC` + `SELECT count(*) as count, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC` ); return ; }; diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx index 5887484f..6c5bb1c5 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx @@ -7,7 +7,7 @@ import { Widget, WidgetBody, WidgetHead } from '@/components/widget'; import { cn } from '@/utils/cn'; import { escape } from 'sqlstring'; -import { chQuery } from '@openpanel/db'; +import { chQuery, TABLE_NAMES } from '@openpanel/db'; interface Props { projectId: string; @@ -21,7 +21,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) { // Days since last event from users // group by days const res = await chQuery( - `SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM events where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51` + `SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM ${TABLE_NAMES.events} where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51` ); const maxValue = Math.max(...res.map((x) => x.count)); @@ -37,7 +37,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) {
( - `SELECT profile_id, count(*) as count from events where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50` + `SELECT profile_id, count(*) as count from ${TABLE_NAMES.events} where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50` ); const profiles = await getProfiles(res.map((r) => r.profile_id)); const list = res.map((item) => { diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx index 23fb1872..a0f183fa 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx @@ -1,7 +1,7 @@ import { subMinutes } from 'date-fns'; import { escape } from 'sqlstring'; -import { chQuery, formatClickhouseDate } from '@openpanel/db'; +import { chQuery, formatClickhouseDate, TABLE_NAMES } from '@openpanel/db'; import type { Coordinate } from './coordinates'; import Map from './map'; @@ -11,7 +11,7 @@ type Props = { }; const RealtimeMap = async ({ projectId }: Props) => { const res = await chQuery( - `SELECT DISTINCT city, longitude as long, latitude as lat FROM events WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC` + `SELECT DISTINCT city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC` ); return ; diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx index 3421ec39..b85700fc 100644 --- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx +++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx @@ -1,6 +1,6 @@ import { escape } from 'sqlstring'; -import { getEvents } from '@openpanel/db'; +import { getEvents, TABLE_NAMES } from '@openpanel/db'; import LiveEvents from './live-events'; @@ -10,7 +10,7 @@ type Props = { }; const RealtimeLiveEventsServer = async ({ projectId, limit = 30 }: Props) => { const events = await getEvents( - `SELECT * FROM events WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`, + `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`, { profile: true, } diff --git a/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx b/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx index 3e724323..8a4c1c6b 100644 --- a/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx +++ b/apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx @@ -5,6 +5,7 @@ import { getCurrentOrganizations, getEvents, getProjectWithClients, + TABLE_NAMES, } from '@openpanel/db'; import OnboardingVerify from './onboarding-verify'; @@ -24,7 +25,7 @@ const Verify = async ({ params: { projectId } }: Props) => { const [project, events] = await Promise.all([ await getProjectWithClients(projectId), getEvents( - `SELECT * FROM events WHERE project_id = ${escape(projectId)} LIMIT 100` + `SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} LIMIT 100` ), ]); diff --git a/apps/dashboard/src/components/projects/project-card.tsx b/apps/dashboard/src/components/projects/project-card.tsx index 9a018667..11715e43 100644 --- a/apps/dashboard/src/components/projects/project-card.tsx +++ b/apps/dashboard/src/components/projects/project-card.tsx @@ -3,7 +3,7 @@ import { shortNumber } from '@/hooks/useNumerFormatter'; import { escape } from 'sqlstring'; import type { IServiceProject } from '@openpanel/db'; -import { chQuery } from '@openpanel/db'; +import { chQuery, TABLE_NAMES } from '@openpanel/db'; import { ChartSSR } from '../chart-ssr'; import { FadeIn } from '../fade-in'; @@ -34,7 +34,7 @@ function ProjectCard({ id, name, organizationSlug }: IServiceProject) { async function ProjectChart({ id }: { id: string }) { const chart = await chQuery<{ value: number; date: string }>( - `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC` + `SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC` ); return ( @@ -62,13 +62,13 @@ async function ProjectMetrics({ id }: { id: string }) { ` SELECT ( - SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} + SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} ) as total, ( - SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month' + SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month' ) as month, ( - SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day' + SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day' ) as day ` ); diff --git a/apps/public/src/app/hero.tsx b/apps/public/src/app/hero.tsx index 615e919e..64354c10 100644 --- a/apps/public/src/app/hero.tsx +++ b/apps/public/src/app/hero.tsx @@ -1,6 +1,6 @@ import { ALink } from '@/components/ui/button'; -import { chQuery } from '@openpanel/db'; +import { chQuery, TABLE_NAMES } from '@openpanel/db'; import AnimatedText from './animated-text'; import { Heading1, Lead2 } from './copy'; @@ -15,7 +15,7 @@ function shortNumber(num: number) { export async function Hero() { const projects = await chQuery<{ project_id: string; count: number }>( - 'SELECT project_id, count(*) as count from events GROUP by project_id order by count()' + `SELECT project_id, count(*) as count from ${TABLE_NAMES.events} GROUP by project_id order by count()` ); const projectCount = projects.length; const eventCount = projects.reduce((acc, { count }) => acc + count, 0); diff --git a/apps/worker/scripts/debug.ts b/apps/worker/scripts/debug.ts index da0a81ed..41817eef 100644 --- a/apps/worker/scripts/debug.ts +++ b/apps/worker/scripts/debug.ts @@ -1,7 +1,7 @@ import { escape } from 'sqlstring'; import type { IClickhouseEvent } from '@openpanel/db'; -import { chQuery, eventBuffer } from '@openpanel/db'; +import { chQuery, eventBuffer, TABLE_NAMES } from '@openpanel/db'; import { sessionsQueue } from '@openpanel/queue/src/queues'; import { redis } from '@openpanel/redis'; @@ -70,7 +70,7 @@ async function debugStalledEvents() { if (stalledEvents.length > 0) { const res = await chQuery( - `SELECT * FROM events WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})` + `SELECT * FROM ${TABLE_NAMES.events} WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})` ); stalledEvents.forEach((item) => { diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.ts index 22a45182..0e1e0d45 100644 --- a/apps/worker/src/jobs/events.create-session-end.ts +++ b/apps/worker/src/jobs/events.create-session-end.ts @@ -1,7 +1,12 @@ import type { Job } from 'bullmq'; import { getTime } from '@openpanel/common'; -import { createEvent, eventBuffer, getEvents } from '@openpanel/db'; +import { + createEvent, + eventBuffer, + getEvents, + TABLE_NAMES, +} from '@openpanel/db'; import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue'; export async function createSessionEnd( @@ -13,12 +18,12 @@ export async function createSessionEnd( ); const sql = ` - SELECT * FROM events + SELECT * FROM ${TABLE_NAMES.events} WHERE session_id = '${payload.sessionId}' AND created_at >= ( SELECT created_at - FROM events + FROM ${TABLE_NAMES.events} WHERE session_id = '${payload.sessionId}' AND name = 'session_start' diff --git a/apps/worker/src/jobs/events.ts b/apps/worker/src/jobs/events.ts index c784f79d..a9422002 100644 --- a/apps/worker/src/jobs/events.ts +++ b/apps/worker/src/jobs/events.ts @@ -1,12 +1,13 @@ import type { Job } from 'bullmq'; import { escape } from 'sqlstring'; -import { chQuery, db } from '@openpanel/db'; +import { chQuery, db, TABLE_NAMES } from '@openpanel/db'; import type { EventsQueuePayload, EventsQueuePayloadCreateSessionEnd, EventsQueuePayloadIncomingEvent, } from '@openpanel/queue'; +import { redis } from '@openpanel/redis'; import { createSessionEnd } from './events.create-session-end'; import { incomingEvent } from './events.incoming-event'; @@ -26,7 +27,7 @@ export async function eventsJob(job: Job) { async function updateEventsCount(projectId: string) { const res = await chQuery<{ count: number }>( - `SELECT count(*) as count FROM events WHERE project_id = ${escape(projectId)}` + `SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}` ); const count = res[0]?.count; if (count) { diff --git a/packages/db/clickhouse_init.sql b/packages/db/clickhouse_init.sql index 6c71cdb0..4339d4a2 100644 --- a/packages/db/clickhouse_init.sql +++ b/packages/db/clickhouse_init.sql @@ -34,6 +34,51 @@ CREATE TABLE IF NOT EXISTS openpanel.events ( ORDER BY (project_id, created_at, profile_id) SETTINGS index_granularity = 8192; +CREATE TABLE IF NOT EXISTS openpanel.events_v2 ( + `id` UUID DEFAULT generateUUIDv4(), + `name` String, + `device_id` String, + `profile_id` String, + `project_id` String, + `session_id` String, + `path` String, + `origin` String, + `referrer` String, + `referrer_name` String, + `referrer_type` String, + `duration` UInt64, + `properties` Map(String, String), + `created_at` DateTime64(3), + `country` String, + `city` String, + `region` String, + `longitude` Nullable(Float32), + `latitude` Nullable(Float32), + `os` String, + `os_version` String, + `browser` String, + `browser_version` String, + -- device: mobile/desktop/tablet + `device` String, + -- brand: (Samsung, OnePlus) + `brand` String, + -- model: (Samsung Galaxy, iPhone X) + `model` String +) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at) +ORDER BY + (project_id, created_at, profile_id) SETTINGS index_granularity = 8192; + +ALTER TABLE + events DROP COLUMN utm_source, + DROP COLUMN utm_medium, + DROP COLUMN utm_campaign, + DROP COLUMN utm_term, + DROP COLUMN utm_content, + DROP COLUMN sdk, + DROP COLUMN sdk_version, + DROP COLUMN client_type, + DROP COLUMN continent; + CREATE TABLE IF NOT EXISTS openpanel.events_bots ( `id` UUID DEFAULT generateUUIDv4(), `project_id` String, diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.ts index c902cf47..78ff5caa 100644 --- a/packages/db/src/buffers/event-buffer.ts +++ b/packages/db/src/buffers/event-buffer.ts @@ -4,7 +4,7 @@ import SuperJSON from 'superjson'; import { deepMergeObjects } from '@openpanel/common'; import { redis, redisPub } from '@openpanel/redis'; -import { ch } from '../clickhouse-client'; +import { ch, TABLE_NAMES } from '../clickhouse-client'; import { transformEvent } from '../services/event.service'; import type { IClickhouseEvent, @@ -30,7 +30,7 @@ const sortOldestFirst = ( export class EventBuffer extends RedisBuffer { constructor() { super({ - table: 'events', + table: TABLE_NAMES.events, redis, }); } @@ -176,7 +176,7 @@ export class EventBuffer extends RedisBuffer { } await ch.insert({ - table: 'events', + table: TABLE_NAMES.events, values: Array.from(itemsToClickhouse).map((item) => item.event), format: 'JSONEachRow', }); diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.ts index 0581fcf1..a49f2384 100644 --- a/packages/db/src/clickhouse-client.ts +++ b/packages/db/src/clickhouse-client.ts @@ -1,6 +1,11 @@ import type { ResponseJSON } from '@clickhouse/client'; import { createClient } from '@clickhouse/client'; +export const TABLE_NAMES = { + events: 'events', + profiles: 'profiles', +}; + export const originalCh = createClient({ url: process.env.CLICKHOUSE_URL, username: process.env.CLICKHOUSE_USER, diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts index a5568bbd..e84a2e85 100644 --- a/packages/db/src/services/chart.service.ts +++ b/packages/db/src/services/chart.service.ts @@ -6,7 +6,7 @@ import type { IGetChartDataInput, } from '@openpanel/validation'; -import { formatClickhouseDate } from '../clickhouse-client'; +import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client'; import { createSqlBuilder } from '../sql-builder'; export function getChartSql({ @@ -94,7 +94,7 @@ export function getChartSql({ if (event.segment === 'one_event_per_user') { sb.from = `( - SELECT DISTINCT ON (profile_id) * from events WHERE ${join( + SELECT DISTINCT ON (profile_id) * from ${TABLE_NAMES.events} WHERE ${join( sb.where, ' AND ' )} diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts index e8e3c0f8..3802929b 100644 --- a/packages/db/src/services/event.service.ts +++ b/packages/db/src/services/event.service.ts @@ -12,6 +12,7 @@ import { chQuery, convertClickhouseDateToJs, formatClickhouseDate, + TABLE_NAMES, } from '../clickhouse-client'; import type { EventMeta, Prisma } from '../prisma-client'; import { db } from '../prisma-client'; @@ -323,7 +324,7 @@ export async function getEventList({ sb.where.projectId = `project_id = ${escape(projectId)}`; if (profileId) { - sb.where.deviceId = `device_id IN (SELECT device_id as did FROM events WHERE profile_id = ${escape(profileId)} group by did)`; + sb.where.deviceId = `device_id IN (SELECT device_id as did FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} group by did)`; } if (startDate && endDate) { @@ -448,7 +449,7 @@ export async function getLastScreenViewFromProfileId({ const [eventInDb] = profileId ? await getEvents( - `SELECT * FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} AND created_at >= now() - INTERVAL 30 MINUTE ORDER BY created_at DESC LIMIT 1` + `SELECT * FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} AND created_at >= now() - INTERVAL 30 MINUTE ORDER BY created_at DESC LIMIT 1` ) : []; diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.ts index fa7cbaae..425375a3 100644 --- a/packages/db/src/services/profile.service.ts +++ b/packages/db/src/services/profile.service.ts @@ -4,7 +4,11 @@ import { toObject } from '@openpanel/common'; import type { IChartEventFilter } from '@openpanel/validation'; import { profileBuffer } from '../buffers'; -import { chQuery, formatClickhouseDate } from '../clickhouse-client'; +import { + chQuery, + formatClickhouseDate, + TABLE_NAMES, +} from '../clickhouse-client'; import { createSqlBuilder } from '../sql-builder'; export type IProfileMetrics = { @@ -18,19 +22,19 @@ export type IProfileMetrics = { export function getProfileMetrics(profileId: string, projectId: string) { return chQuery(` WITH lastSeen AS ( - SELECT max(created_at) as lastSeen FROM events WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} + SELECT max(created_at) as lastSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} ), firstSeen AS ( - SELECT min(created_at) as firstSeen FROM events WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} + SELECT min(created_at) as firstSeen FROM ${TABLE_NAMES.events} WHERE profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} ), screenViews AS ( - SELECT count(*) as screenViews FROM events WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} + SELECT count(*) as screenViews FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} ), sessions AS ( - SELECT count(*) as sessions FROM events WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} + SELECT count(*) as sessions FROM ${TABLE_NAMES.events} WHERE name = 'session_start' AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} ), duration AS ( - SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM events WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} + SELECT avg(duration) as durationAvg, quantilesExactInclusive(0.9)(duration)[1] as durationP90 FROM ${TABLE_NAMES.events} WHERE name = 'session_end' AND duration != 0 AND profile_id = ${escape(profileId)} AND project_id = ${escape(projectId)} ) SELECT lastSeen, firstSeen, screenViews, sessions, durationAvg, durationP90 FROM lastSeen, firstSeen, screenViews,sessions, duration `).then((data) => data[0]!); diff --git a/packages/db/src/services/retention.service.ts b/packages/db/src/services/retention.service.ts index e2bc195d..ed9acb18 100644 --- a/packages/db/src/services/retention.service.ts +++ b/packages/db/src/services/retention.service.ts @@ -1,6 +1,6 @@ import { escape } from 'sqlstring'; -import { chQuery } from '../clickhouse-client'; +import { chQuery, TABLE_NAMES } from '../clickhouse-client'; type IGetWeekRetentionInput = { projectId: string; @@ -15,7 +15,7 @@ WITH SELECT profile_id, max(toWeek(created_at)) AS last_seen - FROM events + FROM ${TABLE_NAMES.events} WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id) GROUP BY profile_id ), @@ -24,7 +24,7 @@ WITH SELECT profile_id, min(toWeek(created_at)) AS first_seen - FROM events + FROM ${TABLE_NAMES.events} WHERE (project_id = ${escape(projectId)}) AND (profile_id != device_id) GROUP BY profile_id ), @@ -79,8 +79,8 @@ export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) { countDistinct(events.profile_id) AS active_users, countDistinct(future_events.profile_id) AS retained_users, (100 * (countDistinct(future_events.profile_id) / CAST(countDistinct(events.profile_id), 'float'))) AS retention - FROM events - LEFT JOIN events AS future_events ON + FROM ${TABLE_NAMES.events} as events + LEFT JOIN ${TABLE_NAMES.events} AS future_events ON events.profile_id = future_events.profile_id AND toStartOfWeek(events.created_at) = toStartOfWeek(future_events.created_at - toIntervalWeek(1)) AND future_events.profile_id != future_events.device_id @@ -140,7 +140,7 @@ export function getRetentionLastSeenSeries({ SELECT max(created_at) AS last_active, profile_id - FROM events + FROM ${TABLE_NAMES.events} WHERE (project_id = ${escape(projectId)}) AND (device_id != profile_id) GROUP BY profile_id ) diff --git a/packages/db/src/sql-builder.ts b/packages/db/src/sql-builder.ts index 7bf0019e..da625fdc 100644 --- a/packages/db/src/sql-builder.ts +++ b/packages/db/src/sql-builder.ts @@ -1,3 +1,5 @@ +import { TABLE_NAMES } from './clickhouse-client'; + export interface SqlBuilderObject { where: Record; having: Record; @@ -15,7 +17,7 @@ export function createSqlBuilder() { const sb: SqlBuilderObject = { where: {}, - from: 'events', + from: TABLE_NAMES.events, select: {}, groupBy: {}, orderBy: {}, diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts index 73cac51c..2fe752c3 100644 --- a/packages/trpc/src/routers/chart.helpers.ts +++ b/packages/trpc/src/routers/chart.helpers.ts @@ -34,6 +34,7 @@ import { getChartSql, getEventFiltersWhereClause, getProfiles, + TABLE_NAMES, } from '@openpanel/db'; import type { FinalChart, @@ -293,7 +294,7 @@ export async function getFunnelData({ const innerSql = `SELECT session_id, windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level - FROM events + FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(startDate)}' AND @@ -306,7 +307,7 @@ export async function getFunnelData({ const [funnelRes, sessionRes] = await Promise.all([ chQuery<{ level: number; count: number }>(sql), chQuery<{ count: number }>( - `SELECT count(name) as count FROM events WHERE project_id = ${escape(projectId)} AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')` + `SELECT count(name) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name = 'session_start' AND (created_at >= '${formatClickhouseDate(startDate)}') AND (created_at <= '${formatClickhouseDate(endDate)}')` ), ]); @@ -409,7 +410,7 @@ export async function getFunnelStep({ const innerSql = `SELECT session_id, windowFunnel(${ONE_DAY_IN_SECONDS})(toUnixTimestamp(created_at), ${funnels.join(', ')}) AS level - FROM events + FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(startDate)}' AND @@ -421,7 +422,7 @@ export async function getFunnelStep({ SELECT DISTINCT e.profile_id as id FROM sessions s - JOIN events e ON s.session_id = e.session_id + JOIN ${TABLE_NAMES.events} e ON s.session_id = e.session_id WHERE s.level = ${step} AND e.project_id = ${escape(projectId)} AND diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts index 707bdea0..2b26ddbb 100644 --- a/packages/trpc/src/routers/chart.ts +++ b/packages/trpc/src/routers/chart.ts @@ -3,7 +3,7 @@ import { escape } from 'sqlstring'; import { z } from 'zod'; import { average, max, min, round, slug, sum } from '@openpanel/common'; -import { chQuery, createSqlBuilder, db } from '@openpanel/db'; +import { chQuery, createSqlBuilder, db, TABLE_NAMES } from '@openpanel/db'; import { zChartInput } from '@openpanel/validation'; import type { FinalChart, @@ -17,7 +17,6 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from '../trpc'; import { getChart, getChartPrevStartEndDate, - getChartSeries, getChartStartEndDate, getFunnelData, getFunnelStep, @@ -28,7 +27,7 @@ export const chartRouter = createTRPCRouter({ .input(z.object({ projectId: z.string() })) .query(async ({ input: { projectId } }) => { const events = await chQuery<{ name: string }>( - `SELECT DISTINCT name FROM events WHERE project_id = ${escape(projectId)}` + `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}` ); return [ @@ -43,7 +42,7 @@ export const chartRouter = createTRPCRouter({ .input(z.object({ event: z.string().optional(), projectId: z.string() })) .query(async ({ input: { projectId, event } }) => { const events = await chQuery<{ keys: string[] }>( - `SELECT distinct mapKeys(properties) as keys from events where ${ + `SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${ event && event !== '*' ? `name = ${escape(event)} AND ` : '' } project_id = ${escape(projectId)};` );