zias/stats
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

prep events partition

Browse Source
This commit is contained in:
Carl-Gerhard Lindesvärd
2024-07-19 22:08:22 +02:00
parent ddc2ce338f
commit 3993b493e3
27 changed files with 136 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/scripts/migrate-origins.ts
View File

@@ -1,8 +1,8 @@
import { ch, chQuery } from '@openpanel/db';
import { ch, chQuery, TABLE_NAMES } from '@openpanel/db';
async function main() {
const projects = await chQuery(
`SELECT distinct project_id FROM events ORDER BY project_id`
`SELECT distinct project_id FROM ${TABLE_NAMES.events} ORDER BY project_id`
);
const withOrigin = [];
@@ -10,10 +10,10 @@ async function main() {
try {
const [eventWithOrigin, eventWithoutOrigin] = await Promise.all([
await chQuery(
`SELECT * FROM events WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`
`SELECT * FROM ${TABLE_NAMES.events} WHERE origin != '' AND project_id = '${project.project_id}' ORDER BY created_at DESC LIMIT 1`
),
await chQuery(
`SELECT * FROM events WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`
`SELECT * FROM ${TABLE_NAMES.events} WHERE origin = '' AND project_id = '${project.project_id}' AND path != '' ORDER BY created_at DESC LIMIT 1`
),
]);
@@ -22,7 +22,7 @@ async function main() {
console.log(`- Origin: ${eventWithOrigin[0].origin}`);
withOrigin.push(project.project_id);
const events = await chQuery(
`SELECT count(*) as count FROM events WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = '${project.project_id}' AND path != '' AND origin = ''`
);
console.log(`🤠🤠🤠🤠 Will update ${events[0]?.count} events`);
await ch.command({

7
apps/api/src/controllers/live.controller.ts
View File

@@ -10,6 +10,7 @@ import {
getEvents,
getLiveVisitors,
getProfileById,
TABLE_NAMES,
transformMinimalEvent,
} from '@openpanel/db';
import { redis, redisPub, redisSub } from '@openpanel/redis';
@@ -28,8 +29,7 @@ export async function testVisitors(
reply: FastifyReply
) {
const events = await getEvents(
`SELECT * FROM events LIMIT 500`
// `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
`SELECT * FROM ${TABLE_NAMES.events} LIMIT 500`
);
const event = events[Math.floor(Math.random() * events.length)];
if (!event) {
@@ -55,8 +55,7 @@ export async function testEvents(
reply: FastifyReply
) {
const events = await getEvents(
`SELECT * FROM events LIMIT 500`
// `SELECT * FROM events WHERE name = 'screen_view' LIMIT 500`
`SELECT * FROM ${TABLE_NAMES.events} LIMIT 500`
);
const event = events[Math.floor(Math.random() * events.length)];
if (!event) {

6
apps/api/src/index.ts
View File

@@ -7,7 +7,7 @@ import Fastify from 'fastify';
import metricsPlugin from 'fastify-metrics';
import { round } from '@openpanel/common';
import { chQuery, db } from '@openpanel/db';
import { chQuery, db, TABLE_NAMES } from '@openpanel/db';
import type { IServiceClient } from '@openpanel/db';
import { eventsQueue } from '@openpanel/queue';
import { redis, redisPub } from '@openpanel/redis';
@@ -101,7 +101,9 @@ const startServer = async () => {
const redisRes = await withTimings(redis.keys('*'));
const dbRes = await withTimings(db.project.findFirst());
const queueRes = await withTimings(eventsQueue.getCompleted());
const chRes = await withTimings(chQuery('SELECT * FROM events LIMIT 1'));
const chRes = await withTimings(
chQuery(`SELECT * FROM ${TABLE_NAMES.events} LIMIT 1`)
);
const status = redisRes && dbRes && queueRes && chRes ? 200 : 500;
reply.status(status).send({

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/events/event-conversions-list/index.tsx
View File

@@ -1,7 +1,7 @@
import withLoadingWidget from '@/hocs/with-loading-widget';
import { escape } from 'sqlstring';
import { db, getEvents } from '@openpanel/db';
import { db, getEvents, TABLE_NAMES } from '@openpanel/db';
import { EventConversionsList } from './event-conversions-list';
@@ -22,7 +22,7 @@ async function EventConversionsListServer({ projectId }: Props) {
}
const events = await getEvents(
`SELECT * FROM events WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,
`SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND name IN (${conversions.map((c) => escape(c.name)).join(', ')}) ORDER BY created_at DESC LIMIT 20;`,
{
profile: true,
meta: true,

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/most-events/index.tsx
View File

@@ -1,7 +1,7 @@
import withLoadingWidget from '@/hocs/with-loading-widget';
import { escape } from 'sqlstring';
import { chQuery } from '@openpanel/db';
import { chQuery, TABLE_NAMES } from '@openpanel/db';
import MostEvents from './most-events';
@@ -12,7 +12,7 @@ type Props = {
const MostEventsServer = async ({ projectId, profileId }: Props) => {
const data = await chQuery<{ count: number; name: string }>(
`SELECT count(*) as count, name FROM events WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC`
`SELECT count(*) as count, name FROM ${TABLE_NAMES.events} WHERE name NOT IN ('screen_view', 'session_start', 'session_end') AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY name ORDER BY count DESC`
);
return <MostEvents data={data} />;
};

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/popular-routes/index.tsx
View File

@@ -1,7 +1,7 @@
import withLoadingWidget from '@/hocs/with-loading-widget';
import { escape } from 'sqlstring';
import { chQuery } from '@openpanel/db';
import { chQuery, TABLE_NAMES } from '@openpanel/db';
import PopularRoutes from './popular-routes';
@@ -12,7 +12,7 @@ type Props = {
const PopularRoutesServer = async ({ projectId, profileId }: Props) => {
const data = await chQuery<{ count: number; path: string }>(
`SELECT count(*) as count, path FROM events WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC`
`SELECT count(*) as count, path FROM ${TABLE_NAMES.events} WHERE name = 'screen_view' AND project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY path ORDER BY count DESC`
);
return <PopularRoutes data={data} />;
};

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/[profileId]/profile-activity/index.tsx
View File

@@ -1,7 +1,7 @@
import withLoadingWidget from '@/hocs/with-loading-widget';
import { escape } from 'sqlstring';
import { chQuery } from '@openpanel/db';
import { chQuery, TABLE_NAMES } from '@openpanel/db';
import ProfileActivity from './profile-activity';
@@ -12,7 +12,7 @@ type Props = {
const ProfileActivityServer = async ({ projectId, profileId }: Props) => {
const data = await chQuery<{ count: number; date: string }>(
`SELECT count(*) as count, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC`
`SELECT count(*) as count, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} and profile_id = ${escape(profileId)} GROUP BY date ORDER BY date DESC`
);
return <ProfileActivity data={data} />;
};

6
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-last-seen/index.tsx
View File

@@ -7,7 +7,7 @@ import { Widget, WidgetBody, WidgetHead } from '@/components/widget';
import { cn } from '@/utils/cn';
import { escape } from 'sqlstring';
import { chQuery } from '@openpanel/db';
import { chQuery, TABLE_NAMES } from '@openpanel/db';
interface Props {
projectId: string;
@@ -21,7 +21,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) {
// Days since last event from users
// group by days
const res = await chQuery<Row>(
`SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM events where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51`
`SELECT age('days',created_at, now()) as days, count(distinct profile_id) as count FROM ${TABLE_NAMES.events} where project_id = ${escape(projectId)} group by days order by days ASC LIMIT 51`
);
const maxValue = Math.max(...res.map((x) => x.count));
@@ -37,7 +37,7 @@ export default async function ProfileLastSeenServer({ projectId }: Props) {
<Tooltip>
<TooltipTrigger asChild>
<div
className={cn('bg-highlight aspect-square w-full shrink-0 rounded')}
className={cn('aspect-square w-full shrink-0 rounded bg-highlight')}
style={{
opacity: calculateRatio(item.count),
}}

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/profiles/profile-top/index.tsx
View File

@@ -7,7 +7,7 @@ import { getProfileName } from '@/utils/getters';
import Link from 'next/link';
import { escape } from 'sqlstring';
import { chQuery, getProfiles } from '@openpanel/db';
import { chQuery, getProfiles, TABLE_NAMES } from '@openpanel/db';
interface Props {
projectId: string;
@@ -18,7 +18,7 @@ async function ProfileTopServer({ organizationSlug, projectId }: Props) {
// Days since last event from users
// group by days
const res = await chQuery<{ profile_id: string; count: number }>(
`SELECT profile_id, count(*) as count from events where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50`
`SELECT profile_id, count(*) as count from ${TABLE_NAMES.events} where profile_id != '' and project_id = ${escape(projectId)} group by profile_id order by count() DESC LIMIT 50`
);
const profiles = await getProfiles(res.map((r) => r.profile_id));
const list = res.map((item) => {

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/map/index.tsx
View File

@@ -1,7 +1,7 @@
import { subMinutes } from 'date-fns';
import { escape } from 'sqlstring';
import { chQuery, formatClickhouseDate } from '@openpanel/db';
import { chQuery, formatClickhouseDate, TABLE_NAMES } from '@openpanel/db';
import type { Coordinate } from './coordinates';
import Map from './map';
@@ -11,7 +11,7 @@ type Props = {
};
const RealtimeMap = async ({ projectId }: Props) => {
const res = await chQuery<Coordinate>(
`SELECT DISTINCT city, longitude as long, latitude as lat FROM events WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
`SELECT DISTINCT city, longitude as long, latitude as lat FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at >= '${formatClickhouseDate(subMinutes(new Date(), 30))}' ORDER BY created_at DESC`
);
return <Map markers={res} />;

4
apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/realtime/realtime-live-events/index.tsx
View File

@@ -1,6 +1,6 @@
import { escape } from 'sqlstring';
import { getEvents } from '@openpanel/db';
import { getEvents, TABLE_NAMES } from '@openpanel/db';
import LiveEvents from './live-events';
@@ -10,7 +10,7 @@ type Props = {
};
const RealtimeLiveEventsServer = async ({ projectId, limit = 30 }: Props) => {
const events = await getEvents(
`SELECT * FROM events WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`,
`SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT ${limit}`,
{
profile: true,
}

3
apps/dashboard/src/app/(onboarding)/onboarding/[projectId]/verify/page.tsx
View File

@@ -5,6 +5,7 @@ import {
getCurrentOrganizations,
getEvents,
getProjectWithClients,
TABLE_NAMES,
} from '@openpanel/db';
import OnboardingVerify from './onboarding-verify';
@@ -24,7 +25,7 @@ const Verify = async ({ params: { projectId } }: Props) => {
const [project, events] = await Promise.all([
await getProjectWithClients(projectId),
getEvents(
`SELECT * FROM events WHERE project_id = ${escape(projectId)} LIMIT 100`
`SELECT * FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} LIMIT 100`
),
]);

10
apps/dashboard/src/components/projects/project-card.tsx
View File

@@ -3,7 +3,7 @@ import { shortNumber } from '@/hooks/useNumerFormatter';
import { escape } from 'sqlstring';
import type { IServiceProject } from '@openpanel/db';
import { chQuery } from '@openpanel/db';
import { chQuery, TABLE_NAMES } from '@openpanel/db';
import { ChartSSR } from '../chart-ssr';
import { FadeIn } from '../fade-in';
@@ -34,7 +34,7 @@ function ProjectCard({ id, name, organizationSlug }: IServiceProject) {
async function ProjectChart({ id }: { id: string }) {
const chart = await chQuery<{ value: number; date: string }>(
`SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM events WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`
`SELECT countDistinct(profile_id) as value, toStartOfDay(created_at) as date FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND name = 'session_start' AND created_at >= now() - interval '1 month' GROUP BY date ORDER BY date ASC`
);
return (
@@ -62,13 +62,13 @@ async function ProjectMetrics({ id }: { id: string }) {
`
SELECT
(
SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)}
SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)}
) as total,
(
SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month'
SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 month'
) as month,
(
SELECT count(DISTINCT profile_id) as count FROM events WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day'
SELECT count(DISTINCT profile_id) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(id)} AND created_at >= now() - interval '1 day'
) as day
`
);

4
apps/public/src/app/hero.tsx
View File

@@ -1,6 +1,6 @@
import { ALink } from '@/components/ui/button';
import { chQuery } from '@openpanel/db';
import { chQuery, TABLE_NAMES } from '@openpanel/db';
import AnimatedText from './animated-text';
import { Heading1, Lead2 } from './copy';
@@ -15,7 +15,7 @@ function shortNumber(num: number) {
export async function Hero() {
const projects = await chQuery<{ project_id: string; count: number }>(
'SELECT project_id, count(*) as count from events GROUP by project_id order by count()'
`SELECT project_id, count(*) as count from ${TABLE_NAMES.events} GROUP by project_id order by count()`
);
const projectCount = projects.length;
const eventCount = projects.reduce((acc, { count }) => acc + count, 0);

4
apps/worker/scripts/debug.ts
View File

@@ -1,7 +1,7 @@
import { escape } from 'sqlstring';
import type { IClickhouseEvent } from '@openpanel/db';
import { chQuery, eventBuffer } from '@openpanel/db';
import { chQuery, eventBuffer, TABLE_NAMES } from '@openpanel/db';
import { sessionsQueue } from '@openpanel/queue/src/queues';
import { redis } from '@openpanel/redis';
@@ -70,7 +70,7 @@ async function debugStalledEvents() {
if (stalledEvents.length > 0) {
const res = await chQuery(
`SELECT * FROM events WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`
`SELECT * FROM ${TABLE_NAMES.events} WHERE id IN (${stalledEvents.map((item) => escape(JSON.parse(item).id)).join(',')})`
);
stalledEvents.forEach((item) => {

11
apps/worker/src/jobs/events.create-session-end.ts
View File

@@ -1,7 +1,12 @@
import type { Job } from 'bullmq';
import { getTime } from '@openpanel/common';
import { createEvent, eventBuffer, getEvents } from '@openpanel/db';
import {
createEvent,
eventBuffer,
getEvents,
TABLE_NAMES,
} from '@openpanel/db';
import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';
export async function createSessionEnd(
@@ -13,12 +18,12 @@ export async function createSessionEnd(
);
const sql = `
SELECT * FROM events
SELECT * FROM ${TABLE_NAMES.events}
WHERE
session_id = '${payload.sessionId}'
AND created_at >= (
SELECT created_at
FROM events
FROM ${TABLE_NAMES.events}
WHERE
session_id = '${payload.sessionId}'
AND name = 'session_start'

5
apps/worker/src/jobs/events.ts
View File

@@ -1,12 +1,13 @@
import type { Job } from 'bullmq';
import { escape } from 'sqlstring';
import { chQuery, db } from '@openpanel/db';
import { chQuery, db, TABLE_NAMES } from '@openpanel/db';
import type {
EventsQueuePayload,
EventsQueuePayloadCreateSessionEnd,
EventsQueuePayloadIncomingEvent,
} from '@openpanel/queue';
import { redis } from '@openpanel/redis';
import { createSessionEnd } from './events.create-session-end';
import { incomingEvent } from './events.incoming-event';
@@ -26,7 +27,7 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {
async function updateEventsCount(projectId: string) {
const res = await chQuery<{ count: number }>(
`SELECT count(*) as count FROM events WHERE project_id = ${escape(projectId)}`
`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`
);
const count = res[0]?.count;
if (count) {